%!PS-Adobe-2.0
%%Creator: dvips 5.526 Copyright 1986, 1994 Radical Eye Software
%%Title: BUG1-1.dvi
%%CreationDate: Mon May  5 17:13:40 1997
%%Pages: 66
%%PageOrder: Ascend
%%BoundingBox: 0 0 612 792
%%EndComments
%DVIPSCommandLine: dvips -o BUG1-1.ps BUG1-1
%DVIPSParameters: dpi=300, comments removed
%DVIPSSource:  TeX output 1997.05.05:1712
%%BeginProcSet: tex.pro
/TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N
/X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72
mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1}
ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale
isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div
hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul
TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get
round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{
statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0
0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn
begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X
array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo
setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx
FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{
pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}
B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup
length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B
/ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type
/stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp
0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2
index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff
ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice
ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{
ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn
/base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1
sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D
}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0
moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add
.99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage
userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook
known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X
/IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for
65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0
0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V
{}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7
getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false}
ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley
false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley
scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave
transform round exch round exch itransform moveto rulex 0 rlineto 0
ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta
0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail}
B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{
3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p
-1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{
3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end
%%EndProcSet
TeXDict begin 40258431 52099146 1000 300 300
(/spandex/homes/rwhaley/Papers/BUG1-1.dvi) @start /Fa
27 127 df<00000000001800000000007C0000000001FC0000000007F8000000001FF000
0000007FC000000001FF0000000007FC000000001FF0000000007FC000000001FF000000
0007FC000000001FF0000000007FC000000001FF0000000007FC000000001FF000000000
7FC000000001FF0000000007FC000000001FF0000000007FC000000000FF0000000000FC
00000000007000000000002E1982962A>8 D<0000000000380000000001FC000000000F
FC000000007FF800000003FFE00000001FFF00000000FFF800000007FFC00000003FFE00
000001FFF00000000FFF800000007FFC00000003FFE00000001FFF000000007FF8000000
00FFC000000000FE00000000007000000000002E12828F2A>16 D<000000000018000000
00007C0000000000FC0000000003F80000000007F0000000001FC0000000003F80000000
00FE0000000001FC0000000007F0000000000FE0000000003F80000000007F0000000001
FC0000000003F8000000000FE0000000001FC0000000007F0000000000FE0000000003F8
0000000007F0000000001FC0000000003F8000000000FE0000000001FC0000000007F000
0000000FE0000000003F80000000007F0000000000FC0000000000F80000000000600000
0000002E20829D2A>I<00000000C000000001E000000003E000000003C000000007C000
00000F800000001F000000001E000000003E000000007C00000000F800000000F0000000
01F000000003E000000007C000000007800000000F800000001F000000003E000000003C
000000007C00000000F800000001F000000001E000000003E000000007C0000000078000
00000F800000001F000000003E000000003C000000007C00000000F800000001F0000000
01E000000003E000000007C00000000F800000000F000000001F000000003E000000007C
000000007800000000F800000000F0000000006000000000232E82AB1F>19
D<0000000000780000000007FC000000007FFC00000007FFF80000007FFFC000000FFFFC
000000FFFFC000000FFFFC000000FFFFC000000FFFF80000007FFF80000000FFF8000000
00FF80000000007800000000002E0E828B2A>24 D<000040000000C0000001E0000003E0
000007F0000007F000000FF800001FF800003FFC00007FFC0000FFFE0001FFFE0003FFFF
0007FFFF000FFFFF801FFFFF803FFC00007C0000008000000019137F922A>I<00000200
000E00003E0001FE0007FE003FFE01FFFE0FFFFEFFFFFEFFFFFE0FFFFE01FFFE003FFE00
07FE0001FE00003E00000E00000217127E882A>27 D<0000100000003000000078000000
F8000001F8000007FC00000FFC00001FFC00003FFE00007FFE0001FFFE0003FFFF000FFF
FF001FFFFF007FFFFF80C0003F800000008019117F8F2A>41 D<00010000000180000003
C0000007C0000007E000000FF000000FF000001FF800003FFC00007FFC00007FFE0000FF
FF0001FFFF0003FFFF8003FFFFC007FFF0000FFE00001FE000003E000000700000008000
00001A1580942A>43 D<800000E00000F80000FF0000FFC000FFF800FFFF00FFFFE0FFFF
FEFFFFFEFFFFE0FFFF00FFF800FFC000FF0000F80000E0000080000017126F882A>45
D<01800001C00001F00001F80001FC0003FF0003FF8003FFC003FFF007FFF807FFFC07FF
F007FFC00FFF800FFE000FF8001FF0001FC0001F00003E00003C00007000006000004000
00800000161980981F>47 D<000008000000180000003C0000007C000000FC000003FC00
0007FE00000FFE00003FFE00007FFE0001FFFF0007FFFF000FFFFF003FFFFF00FFFFFF80
000FFF8000001F800000008019127F8E2A>57 D<FFFFC07FFF807FFF803FFF003FFF001F
FE001FFE001FFE000FFC000FFC0007F80007F80007F80003F00003F00003F00001E00001
E00001E00000C00000C00000C00000C000121789982A>63 D<700000000000FC00000000
00FF00000000007FC0000000001FF00000000007FC0000000001FF00000000007FC00000
00001FF00000000007FC0000000001FF00000000007FC0000000001FF00000000007FC00
00000001FF00000000007FC0000000001FF00000000007FC0000000001FF00000000007F
C0000000001FF00000000007F80000000001FC00000000007C0000000000182E1982962A
>72 D<60000000F0000000F8000000780000007C0000003E0000001E0000001F0000000F
8000000780000007C0000003E0000001E0000001F0000000F8000000780000007C000000
3E0000001E0000001F0000000F8000000780000007C0000003E0000001E0000001F00000
00F8000000780000007C0000003E0000001E0000001F0000000F8000000780000007C000
0003E0000001E0000001F0000000F8000000780000007C0000003E0000001E0000001F00
00000F00000006202E82AB1C>74 D<700000000000FE0000000000FFC0000000007FF800
0000001FFF0000000003FFE0000000007FFC000000000FFF8000000001FFF0000000003F
FE0000000007FFC000000000FFF8000000001FFF0000000003FFE0000000007FF8000000
000FFC0000000001FC0000000000382E12828F2A>80 D<600000000000F80000000000FC
00000000007F00000000003F80000000000FE00000000007F00000000001FC0000000000
FE00000000003F80000000001FC00000000007F00000000003F80000000000FE00000000
007F00000000001FC0000000000FE00000000003F80000000001FC00000000007F000000
00003F80000000000FE00000000007F00000000001FC0000000000FE00000000003F8000
0000001FC00000000007F00000000003F80000000000FC00000000007C0000000000182E
20829D2A>I<780000000000FF8000000000FFF8000000007FFF800000000FFFF8000000
00FFFFC00000000FFFFC00000000FFFFC00000000FFFFC000000007FFFC000000007FFF8
000000007FFC0000000007FC0000000000782E0E828B2A>88 D<800000007C0000003FFC
00001FFFFF800FFFFF8007FFFF0003FFFF0001FFFE0000FFFE00007FFC00003FFC00001F
F800000FF8000007F0000007F0000003E0000001E0000000C0000000400019137F942A>
I<600000000000F80000000000FC00000000007E00000000003F80000000000FC0000000
0007E00000000003F80000000000FC00000000007E00000000003F80000000000FC00000
000007E00000000003F80000000000FC00000000007E00000000003F80000000000FC000
00000007F00000000003F80000000000FC00000000007F00000000001F80000000000FC0
0000000007F00000000001F80000000000FC00000000007F00000000001F80000000000F
C00000000007F00000000001F80000000000FC00000000007C0000000000182E2382A02A
>I<000200000600001E00003E0000FE0001FE0007FF000FFF003FFF007FFF00FFFF003F
FF801FFF8007FF8001FF8000FFC0007FC0001FC0000FC00007E00001E00000E000007000
0030000010000008151A79991C>94 D<010000000180000003C0000003E0000007F00000
07F000000FF800000FFC00001FFE00001FFF00003FFF80003FFFC0007FFFE0007FFFF000
FFFFF800FFFFFC00001FFE0000001F0000000080191370922A>106
D<80000000700000003E0000001FE000000FFE000007FFF00003FFFFC003FFFF8001FFFF
0000FFFF00007FFE00007FFC00003FFC00001FF800000FF000000FF0000007E0000007C0
000003C00000018000000100001A15809B2A>I<04000000060000000F0000000F800000
0FC000001FF000001FF800001FFC00003FFE00003FFF00003FFFC0007FFFE0007FFFF800
7FFFFC00FFFFFF00FE000180800000001911708F2A>113 D<002000000060000000F000
0000F8000001F8000003FC000003FC000007FE00000FFF00000FFF80001FFF80003FFFC0
003FFFE0007FFFF000FFFFF00003FFF800001FFC000001FE0000001F0000000380000000
401A1570942A>115 D<080000000C0000001E0000001F0000001F8000001FE000003FF0
00003FF800003FFE00003FFF00007FFFC0007FFFF0007FFFF8007FFFFE00FFFFFF80FFF8
0000FC000000800000001912708E2A>122 D<002000000060000000F0000000F0000001
F8000003FC000007FC000007FE00000FFE00001FFF00003FFF80003FFF80007FFFC000FF
FFC000FFFFE00007FFF000007FF8000007F8000000FC0000001E00000007000000008019
166F952A>126 D E /Fb 14 28 df<7C0000FF8000FFC0007FF00007F80000FC00007E00
001F00000F800007800007C00003E00001E00001E00001F00000F00000F00000F00000F0
0000701414908321>4 D<0003F0001FF8003FF800FFF001FE0003F00007E0000F80001F
00001E00003E00007C0000780000780000F80000F00000F00000F00000F0000070000015
14808321>7 D<7F00000000FFF0000000FFFE0000007FFF80000001FFE00000000FF800
000003FE00000000FF000000003F800000000FC000000003E000000001F000000000F800
0000007C000000003E000000001E000000001F000000000F000000000F80000000078000
000007C000000003C000000003E000000001E000000001E000000001F000000000F00000
0000F000000000F000000000F80000000078000000007800000000780000000078000000
0078000000007800000000302525A18342>12 D<00000000300000000078000000007800
0000007800000000780000000078000000007800000000F800000000F000000000F00000
0000F000000000F000000001F000000001E000000003E000000003C000000007C0000000
07800000000F800000000F000000001F000000001E000000003E000000007C00000000F8
00000001F000000003E00000000FC00000003F80000000FF00000003FE0000000FF80000
01FFE000007FFF800000FFFE000000FFF80000007F000000002525A1A442>I<60000000
00F000000000F000000000F000000000F000000000F000000000F000000000F800000000
78000000007800000000780000000078000000007C000000003C000000003E000000001E
000000001F000000000F000000000F80000000078000000007C000000003C000000003E0
00000001F000000000F8000000007C000000003E000000001F800000000FE000000007F8
00000003FE00000000FF800000003FFC0000000FFFF0000003FFF8000000FFF800000007
F0252580A442>I<00000007F0000000FFF8000003FFF800000FFFF000003FFC000000FF
80000003FE00000007F80000000FE00000001F800000003E000000007C00000000F80000
0001F000000003E000000003C000000007C000000007800000000F800000000F00000000
1F000000001E000000003E000000003C000000007C000000007C00000000780000000078
000000007800000000F800000000F000000000F000000000F000000000F000000000F000
000000F00000000060000000002525808342>I<7FC00000000000FFFE0000000000FFFF
C0000000007FFFF800000000007FFF000000000003FFC000000000007FE000000000000F
F8000000000001FE0000000000007F0000000000003F8000000000000FE0000000000003
F0000000000001F8000000000000FC0000000000003E0000000000001F0000000000000F
80000000000007C0000000000003E0000000000001E0000000000001F0000000000000F8
0000000000007C0000000000003C0000000000003E0000000000001E0000000000001F00
00000000000F8000000000000780000000000007C0000000000003C0000000000003C000
0000000003E0000000000001E0000000000001E0000000000001F0000000000000F00000
00000000F0000000000000F8000000000000780000000000007800000000000078000000
000000780000000000007C0000000000003C0000000000003C0000000000003C00000000
00003C0000000000003C0000000000003C0000000000003C0000000000003C0000000000
00183636B28364>20 D<000000000000180000000000003C0000000000003C0000000000
003C0000000000003C0000000000003C0000000000003C0000000000003C000000000000
3C0000000000007C00000000000078000000000000780000000000007800000000000078
000000000000F8000000000000F0000000000000F0000000000001F0000000000001E000
0000000001E0000000000003E0000000000003C0000000000003C0000000000007C00000
000000078000000000000F8000000000001F0000000000001E0000000000003E00000000
00003C0000000000007C000000000000F8000000000001F0000000000001E00000000000
03E0000000000007C000000000000F8000000000001F0000000000003E000000000000FC
000000000001F8000000000003F000000000000FE000000000003F8000000000007F0000
00000001FE00000000000FF800000000007FE00000000003FFC0000000007FFF00000000
7FFFF800000000FFFFC000000000FFFE00000000007FC000000000003636B2B564>I<60
000000000000F0000000000000F0000000000000F0000000000000F0000000000000F000
0000000000F0000000000000F0000000000000F0000000000000F8000000000000780000
000000007800000000000078000000000000780000000000007C0000000000003C000000
0000003C0000000000003E0000000000001E0000000000001E0000000000001F00000000
00000F0000000000000F0000000000000F8000000000000780000000000007C000000000
0003E0000000000001E0000000000001F0000000000000F0000000000000F80000000000
007C0000000000003E0000000000001E0000000000001F0000000000000F800000000000
07C0000000000003E0000000000001F0000000000000FC0000000000007E000000000000
3F0000000000001FC0000000000007F0000000000003F8000000000001FE000000000000
7FC000000000001FF800000000000FFF000000000003FFF800000000007FFFF800000000
0FFFFC0000000001FFFC00000000000FF8363680B564>I<00000000000FF80000000001
FFFC000000000FFFFC000000007FFFF800000003FFF8000000000FFF00000000001FF800
000000007FC00000000001FE000000000003F8000000000007F000000000001FC0000000
00003F0000000000007E000000000000FC000000000001F0000000000003E00000000000
07C000000000000F8000000000001F0000000000001E0000000000003E0000000000007C
000000000000F8000000000000F0000000000001F0000000000001E0000000000003E000
0000000007C00000000000078000000000000F8000000000000F0000000000000F000000
0000001F0000000000001E0000000000001E0000000000003E0000000000003C00000000
00003C0000000000007C0000000000007800000000000078000000000000780000000000
0078000000000000F8000000000000F0000000000000F0000000000000F0000000000000
F0000000000000F0000000000000F0000000000000F0000000000000F000000000000060
0000000000003636808364>I<7FC0000000000000FFFF000000000000FFFFE000000000
007FFFFC0000000000007FFF80000000000001FFE00000000000003FF800000000000007
FE00000000000000FF800000000000003FC00000000000000FF000000000000003F80000
0000000000FC000000000000007F000000000000001F800000000000000FC00000000000
0007E000000000000001F000000000000000F8000000000000007C000000000000003E00
0000000000001F000000000000000F80000000000000078000000000000007C000000000
000003E000000000000001F000000000000000F000000000000000F8000000000000007C
000000000000003C000000000000003E000000000000001E000000000000001F00000000
0000000F000000000000000F80000000000000078000000000000007C000000000000003
C000000000000003C000000000000003E000000000000001E000000000000001E0000000
00000001F000000000000000F000000000000000F000000000000000F800000000000000
780000000000000078000000000000007800000000000000780000000000000078000000
000000007C000000000000003C000000000000003C000000000000003C00000000000000
3C000000000000003C000000000000003C000000000000003C000000000000003C000000
00000000183E3EBA8374>I<0000000000000018000000000000003C000000000000003C
000000000000003C000000000000003C000000000000003C000000000000003C00000000
0000003C000000000000003C000000000000007C00000000000000780000000000000078
00000000000000780000000000000078000000000000007800000000000000F800000000
000000F000000000000000F000000000000001F000000000000001E000000000000001E0
00000000000003E000000000000003C000000000000003C000000000000007C000000000
000007800000000000000F800000000000000F000000000000001F000000000000001E00
0000000000003E000000000000003C000000000000007C00000000000000F80000000000
0000F000000000000001F000000000000003E000000000000007C0000000000000078000
00000000000F800000000000001F000000000000003E000000000000007C000000000000
00F800000000000001F000000000000007E00000000000000FC00000000000001F800000
000000007F00000000000000FC00000000000003F80000000000000FF00000000000003F
C0000000000000FF80000000000007FE0000000000003FF8000000000001FFE000000000
007FFF80000000007FFFFC0000000000FFFFE00000000000FFFF0000000000007FC00000
000000003E3EBABD74>I<6000000000000000F000000000000000F000000000000000F0
00000000000000F000000000000000F000000000000000F000000000000000F000000000
000000F000000000000000F8000000000000007800000000000000780000000000000078
00000000000000780000000000000078000000000000007C000000000000003C00000000
0000003C000000000000003E000000000000001E000000000000001E000000000000001F
000000000000000F000000000000000F000000000000000F800000000000000780000000
00000007C000000000000003C000000000000003E000000000000001E000000000000001
F000000000000000F000000000000000F8000000000000007C000000000000003C000000
000000003E000000000000001F000000000000000F800000000000000780000000000000
07C000000000000003E000000000000001F000000000000000F8000000000000007C0000
00000000003E000000000000001F800000000000000FC000000000000007E00000000000
0003F800000000000000FC000000000000007F000000000000003FC00000000000000FF0
00000000000007FC00000000000001FF800000000000007FF00000000000001FFE000000
00000007FFF8000000000000FFFFF800000000001FFFFC000000000003FFFC0000000000
000FF83E3E80BD74>I<0000000000000FF8000000000003FFFC00000000001FFFFC0000
000000FFFFF80000000007FFF800000000001FFE0000000000007FF0000000000001FF80
000000000007FC0000000000000FF00000000000003FC00000000000007F000000000000
00FC00000000000003F800000000000007E00000000000000FC00000000000001F800000
000000003E000000000000007C00000000000000F800000000000001F000000000000003
E000000000000007C000000000000007800000000000000F800000000000001F00000000
0000003E000000000000003C000000000000007C00000000000000F800000000000000F0
00000000000001F000000000000001E000000000000003E000000000000003C000000000
000007C000000000000007800000000000000F800000000000000F000000000000000F00
0000000000001F000000000000001E000000000000001E000000000000003E0000000000
00003C000000000000003C000000000000007C0000000000000078000000000000007800
000000000000780000000000000078000000000000007800000000000000F80000000000
0000F000000000000000F000000000000000F000000000000000F000000000000000F000
000000000000F000000000000000F000000000000000F000000000000000600000000000
00003E3E808374>I E /Fc 3 115 df<780018001800300030003000370078C060406060
6060C0C0C0C0C0C0418063003C000B117E900E>98 D<1C70278C2604260606060C0C0C0C
0C0C0C181E3019C01800180030003000FC000F10808A10>112 D<73C05C209860980018
003000300030003000600060000B0B7E8A0E>114 D E /Fd 46 126
df<30787C3C1C1C1C1C3878F0E040060D789816>39 D<387C7E7E3E0E1E1C78F060070B
798416>44 D<FFFF80FFFF80FFFF8011037E8D16>I<70F8F8F8700505788416>I<000180
000380000380000700000700000E00000E00001C00001C00003800003800007000007000
00E00000E00001C00001C0000380000380000700000700000E00000E00001C00001C0000
380000380000700000700000E00000E00000C0000011207E9C16>I<03E0000FF8001FFC
001E3C00380E00780F00700700700700E00380E00380E00380E00380E00380E00380E003
80E00380F00780700700700700780F003C1E001E3C001FFC000FF80003E00011197E9816
>I<0300070007000F001F00FF00FF00E700070007000700070007000700070007000700
07000700070007000700FFF8FFF8FFF80D197B9816>I<7FFFC0FFFFE0FFFFE000000000
0000000000000000000000FFFFE0FFFFE07FFFC0130B7F9116>61
D<00E00001F00001F00001B00001B00003B80003B80003B800031800071C00071C00071C
00071C00071C000E0E000E0E000FFE000FFE001FFF001C07001C07001C07007E0FC0FF1F
E07E0FC013197F9816>65 D<7FF800FFFE007FFF001C07001C07801C03801C03801C0380
1C03801C07001FFF001FFE001FFE001C1F001C03801C03C01C01C01C01C01C01C01C01C0
1C03C01C07807FFF80FFFF007FFC0012197F9816>I<01F18007FB800FFF801F0F803C07
80380380700380700380F00000E00000E00000E00000E00000E00000E00000E00000F000
007003807003803803803C07001F0F000FFE0007FC0001F00011197E9816>I<7FF800FF
FE007FFF001C0F001C07801C03C01C01C01C01C01C01E01C00E01C00E01C00E01C00E01C
00E01C00E01C00E01C00E01C01C01C01C01C03C01C07801C0F807FFF00FFFE007FF80013
19809816>I<7FFFC0FFFFC07FFFC01C01C01C01C01C01C01C01C01C00001C00001C1C00
1C1C001FFC001FFC001FFC001C1C001C1C001C00001C00E01C00E01C00E01C00E01C00E0
7FFFE0FFFFE07FFFE013197F9816>I<7FFFE0FFFFE07FFFE01C00E01C00E01C00E01C00
E01C00001C00001C1C001C1C001FFC001FFC001FFC001C1C001C1C001C00001C00001C00
001C00001C00001C00007F8000FFC0007F800013197F9816>I<01F18007FB800FFF801F
0F803C0780380380700380700380F00000E00000E00000E00000E00000E01FC0E01FE0E0
1FC0F003807003807007803807803C07801E0F800FFF8007FB8001F38013197F9816>I<
7FFCFFFE7FFC038003800380038003800380038003800380038003800380038003800380
03800380038003807FFCFFFE7FFC0F197D9816>73 D<FF8000FFC000FF80001C00001C00
001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00
001C00401C00E01C00E01C00E01C00E0FFFFE0FFFFE0FFFFE013197F9816>76
D<FC07E0FE0FE0FE0FE03A0B803B1B803B1B803B1B803B1B803B1B803BBB8039B38039B3
8039B38039B38039F38038E38038E380380380380380380380380380380380FE0FE0FE0F
E0FE0FE013197F9816>I<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01C
E1C01CE1C01C61C01C71C01C71C01C71C01C31C01C39C01C39C01C19C01C1DC01C0DC01C
0DC01C0DC07F07C0FF87C07F03C01519809816>I<7FF800FFFE007FFF001C0F801C0380
1C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FF8001C00001C0000
1C00001C00001C00001C00001C00007F0000FF80007F000012197F9816>80
D<7FF000FFFC007FFE001C0F001C07801C03801C03801C03801C03801C07801C0F001FFE
001FFC001FFE001C0E001C07001C07001C07001C07001C07101C07381C07387F03F0FF83
F07F01E01519809816>82 D<07E3001FFF003FFF00781F00F00700E00700E00700E00000
F000007800003F80001FF00007FC0000FE00000F00000700000380000380600380E00380
E00700F80F00FFFE00FFFC00C7F00011197E9816>I<7FFFE0FFFFE0FFFFE0E0E0E0E0E0
E0E0E0E0E0E0E000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0
0000E00000E00000E00000E00000E00007FC000FFE0007FC0013197F9816>I<7E03F0FF
07F87E03F03800E03800E03800E03800E01C01C01C01C01C01C01C71C01CF9C01CF9C01C
D9C01CD9C00CD9800CD9800CD9800DDD800D8D800D8D800D8D80078F00078F0007070015
19809816>87 D<FFFF80FFFF80FFFF8011037E7E16>95 D<1FE0003FF0007FF800783C00
300E00000E00000E0003FE001FFE003E0E00700E00E00E00E00E00E00E00783E007FFFE0
3FE7E00F83E013127E9116>97 D<FC0000FC0000FC00001C00001C00001C00001C00001C
7C001DFE001FFF001F07801E03C01C01C01C00E01C00E01C00E01C00E01C00E01C00E01C
01C01E03C01F07801FFF001DFE000C780013197F9816>I<03F80FFC1FFE3C1E780C7000
E000E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>I<003F00003F
00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F00700700E007
00E00700E00700E00700E00700E00700700F00780F003C1F001FFFE00FF7E007C7E01319
7F9816>I<03F0000FFC001FFE003C0F00780700700380E00380FFFF80FFFF80FFFF80E0
0000E000007003807803803E07801FFF0007FE0001F80011127E9116>I<001E00007F00
01FF8001C7800383000380000380007FFF00FFFF00FFFF00038000038000038000038000
0380000380000380000380000380000380000380000380007FFC00FFFE007FFC0011197F
9816>I<03E3C007F7E00FFFE01C1CC0380E00380E00380E00380E00380E001C1C000FF8
001FF0001BE0003800001800001FFC001FFF003FFF807003C0E000E0E000E0E000E0E000
E07001C07C07C03FFF800FFE0003F800131C7F9116>I<03000780078003000000000000
000000FF80FF80FF80038003800380038003800380038003800380038003800380FFFCFF
FEFFFC0F1A7C9916>105 D<7E0000FE00007E00000E00000E00000E00000E00000E7FE0
0E7FE00E7FE00E0F000E1E000E3C000E78000EF0000FF0000FF8000FBC000F1E000E0E00
0E07000E07807F87F0FFCFF07F87F01419809816>107 D<7FC000FFC0007FC00001C000
01C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000
01C00001C00001C00001C00001C00001C0007FFF00FFFF807FFF0011197E9816>I<F9C3
80FFEFC0FFFFE03C78E03C78E03870E03870E03870E03870E03870E03870E03870E03870
E03870E03870E0FE7CF8FE7CF8FE7CF81512809116>I<FC7800FDFC00FFFE001F0F001E
07001C07001C07001C07001C07001C07001C07001C07001C07001C07001C0700FF8FE0FF
9FE0FF8FE013127F9116>I<03E0000FF8001FFC003C1E00780F00700700E00380E00380
E00380E00380E00380F00780700700780F003C1E001FFC000FF80003E00011127E9116>
I<FC7C00FDFE00FFFF001F07801E03C01C01C01C00E01C00E01C00E01C00E01C00E01C00
E01C01C01E03C01F07801FFF001DFE001C78001C00001C00001C00001C00001C00001C00
00FF8000FF8000FF8000131B7F9116>I<7F0FC0FF3FE07F7FE007F04007C00007800007
80000700000700000700000700000700000700000700000700007FFC00FFFC007FFC0013
127F9116>114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE00EF00E
F81EFFFCFFF8C7E00F127D9116>I<0300000700000700000700000700007FFF00FFFF00
FFFF00070000070000070000070000070000070000070000070100070380070380070380
07870003FE0001FC0000F80011177F9616>I<FC3F00FC3F00FC3F001C07001C07001C07
001C07001C07001C07001C07001C07001C07001C07001C07001C1F001FFFE00FF7E007C7
E013127F9116>I<7F1FC07F3FC07F1FC00F1E00073C0003B80003F00001F00000E00001
E00001F00003B800073C00071C000E0E007F1FC0FF3FE07F1FC013127F9116>120
D<001F80007F8000FF8001E00001C00001C00001C00001C00001C00001C00001C00001C0
0001C00003C0007F8000FF0000FF00007F800003C00001C00001C00001C00001C00001C0
0001C00001C00001C00001C00001E00000FF80007F80001F8011207E9C16>123
D<7C0000FF0000FF800003C00001C00001C00001C00001C00001C00001C00001C00001C0
0001C00001E00000FF00007F80007F8000FF0001E00001C00001C00001C00001C00001C0
0001C00001C00001C00001C00003C000FF8000FF00007C000011207E9C16>125
D E /Fe 1 51 df<7FFFFF80FFFFFF80C0000180C0000180C0000180C0000180C0000180
C0000180C0000180C0000180C0000180C0000180C0000180C0000180C0000180C0000180
C0000180C0000180C0000180C0000180C0000180C0000180C0000180FFFFFF807FFFFF80
19197C9B22>50 D E /Ff 5 113 df<0102040C1818303070606060E0E0E0E0E0E0E0E0
E0E060606070303018180C04020108227D980E>40 D<8040203018180C0C0E0606060707
07070707070707070606060E0C0C18183020408008227E980E>I<06000E00FE000E000E
000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B157D
9412>49 D<0F8030E040708030C038E0384038003800700070006000C00180030006000C
08080810183FF07FF0FFF00D157E9412>I<F9F0003E1C00380600380700380300380380
380380380380380380380300380700380E003E1C0039F000380000380000380000380000
380000FE000011147F8D14>112 D E /Fg 1 66 df<00000080000000018000000001C0
00000003C000000007C000000007C00000000BC00000000BC000000013C000000033C000
000023C000000043E000000043E000000081E000000101E000000101E000000201E00000
0201E000000401E000000801E000000801E000001001F000001FFFF000002000F0000040
00F000004000F000008000F000008000F000010000F000020000F000020000F800040000
78000C000078001E0000F800FF800FFF8021237EA225>65 D E /Fh
1 65 df<800000000000C000000000006000000000003000000000001800000000000C00
0000000006000000000003000000000001800000000000C0000000000060000000000030
00000000001800000000000C000000000006000000000003000000000001800000000000
C000000000006000000000003000000000001800000000000C0000000000060000000000
03000000000001800000000000C000000000006000000000003000000000001800000000
000C000000000006000000000003000000000001800000000000C0000000000060000000
00003000000000001800000000000C000000000006000000000003000000000001800000
000000C00000000000600000000000202B2C80AA2A>64 D E /Fi
1 1 df<FFFFFF80FFFFFF8019027D8A20>0 D E /Fj 2 111 df<381F81F04E20C61846
40E81C4680F01C8F00F01C8E00E01C0E00E01C0E00E01C1C01C0381C01C0381C01C0381C
01C0703803807138038071380380E1380380E2700700643003003820127E9124>109
D<3C1F004E61804681C04701C08F01C08E01C00E01C00E01C01C03801C03801C03801C07
00380710380710380E10380E2070064030038014127E9119>I E
/Fk 62 123 df<0007F80FF000007FFE7FFC0001F80FF80E0003E00FE01F0007C01FC03F
000F801F803F000F801F803F000F800F801E000F800F800C000F800F8000000F800F8000
000F800F8000000F800F800000FFFFFFFFFF00FFFFFFFFFF000F800F801F000F800F801F
000F800F801F000F800F801F000F800F801F000F800F801F000F800F801F000F800F801F
000F800F801F000F800F801F000F800F801F000F800F801F000F800F801F000F800F801F
000F800F801F000F800F801F000F800F801F000F800F801F007FF07FF0FFE07FF07FF0FF
E02B237FA22F>14 D<00180030006000C001C00380070007000E001E001C003C003C003C
007800780078007800F800F000F000F000F000F000F000F000F000F000F000F000F80078
007800780078003C003C003C001C001E000E0007000700038001C000C00060003000180D
317BA416>40 D<C0006000300018001C000E0007000700038003C001C001E001E001E000
F000F000F000F000F80078007800780078007800780078007800780078007800F800F000
F000F000F001E001E001E001C003C00380070007000E001C00180030006000C0000D317D
A416>I<387CFEFFFF7F3B03030306060C1C18702008117C8610>44
D<FFFCFFFCFFFCFFFC0E047F8C13>I<387CFEFEFE7C3807077C8610>I<00180000780001
F800FFF800FFF80001F80001F80001F80001F80001F80001F80001F80001F80001F80001
F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001
F80001F80001F80001F8007FFFE07FFFE013207C9F1C>49 D<03FC000FFF003C1FC07007
E07C07F0FE03F0FE03F8FE03F8FE01F87C01F83803F80003F80003F00003F00007E00007
C0000F80001F00003E0000380000700000E01801C0180380180700180E00380FFFF01FFF
F03FFFF07FFFF0FFFFF0FFFFF015207D9F1C>I<00FE0007FFC00F07E01E03F03F03F03F
81F83F81F83F81F81F03F81F03F00003F00003E00007C0001F8001FE0001FF000007C000
01F00001F80000FC0000FC3C00FE7E00FEFF00FEFF00FEFF00FEFF00FC7E01FC7801F81E
07F00FFFC001FE0017207E9F1C>I<0000E00001E00003E00003E00007E0000FE0001FE0
001FE00037E00077E000E7E001C7E00187E00307E00707E00E07E00C07E01807E03807E0
7007E0E007E0FFFFFFFFFFFF0007E00007E00007E00007E00007E00007E00007E000FFFF
00FFFF18207E9F1C>I<3000203E01E03FFFC03FFF803FFF003FFE003FF80033C0003000
0030000030000030000031FC0037FF003E0FC03807E03003E00003F00003F00003F80003
F83803F87C03F8FE03F8FE03F8FC03F0FC03F07007E03007C03C1F800FFF0003F8001520
7D9F1C>I<001F8000FFE003E07007C0F00F01F81F01F83E01F83E01F87E00F07C00007C
0000FC0800FC7FC0FCFFE0FD80F0FF00F8FE007CFE007CFC007EFC007EFC007EFC007E7C
007E7C007E7C007E3C007C3E007C1E00F80F00F00783E003FFC000FF0017207E9F1C>I<
6000007800007FFFFE7FFFFE7FFFFC7FFFF87FFFF87FFFF0E00060E000C0C00180C00300
C00300000600000C00001C0000180000380000780000780000F00000F00000F00001F000
01F00001F00003F00003F00003F00003F00003F00003F00003F00001E00017227DA11C>
I<00FE0003FFC00703E00E00F01C00F01C00783C00783E00783F00783F80783FE0F01FF9
E01FFFC00FFF8007FFC003FFE007FFF01E7FF83C1FFC7807FC7801FEF000FEF0003EF000
1EF0001EF0001EF8001C7800383C00381F00F00FFFC001FF0017207E9F1C>I<01FE0007
FF800F83E01E01F03E00F07C00F87C0078FC007CFC007CFC007CFC007EFC007EFC007EFC
007E7C00FE7C00FE3E01FE1E037E0FFE7E07FC7E00207E00007C00007C1E007C3F00F83F
00F83F00F03F01E01E03C01C0F800FFE0003F80017207E9F1C>I<000070000000007000
000000F800000000F800000000F800000001FC00000001FC00000003FE00000003FE0000
0003FE000000067F000000067F0000000C7F8000000C3F8000000C3F800000181FC00000
181FC00000301FE00000300FE00000700FF000006007F000006007F00000C007F80000FF
FFF80001FFFFFC00018001FC00018001FC00030001FE00030000FE00070000FF00060000
7F000600007F00FFE007FFF8FFE007FFF825227EA12A>65 D<FFFFFF8000FFFFFFF00007
F001F80007F000FC0007F0007E0007F0007E0007F0007F0007F0007F0007F0007F0007F0
007F0007F0007F0007F0007E0007F000FE0007F000FC0007F003F80007FFFFF00007FFFF
F00007F001FC0007F0007E0007F0003F0007F0003F8007F0001F8007F0001FC007F0001F
C007F0001FC007F0001FC007F0001FC007F0001FC007F0003F8007F0003F8007F0007F00
07F001FE00FFFFFFF800FFFFFFC00022227EA128>I<0003FE0080001FFF818000FF01E3
8001F8003F8003E0001F8007C0000F800F800007801F800007803F000003803F00000380
7F000001807E000001807E00000180FE00000000FE00000000FE00000000FE00000000FE
00000000FE00000000FE00000000FE000000007E000000007E000001807F000001803F00
0001803F000003801F800003000F8000030007C000060003F0000C0001F800380000FF00
F000001FFFC0000003FE000021227DA128>I<FFFFFF8000FFFFFFF00007F003FC0007F0
007E0007F0003F0007F0001F8007F0000FC007F00007E007F00007E007F00007F007F000
03F007F00003F007F00003F007F00003F807F00003F807F00003F807F00003F807F00003
F807F00003F807F00003F807F00003F807F00003F807F00003F007F00003F007F00003F0
07F00007E007F00007E007F0000FC007F0001F8007F0003F0007F0007E0007F003FC00FF
FFFFF000FFFFFF800025227EA12B>I<FFFFFFFCFFFFFFFC07F000FC07F0003C07F0001C
07F0000C07F0000E07F0000E07F0000607F00C0607F00C0607F00C0607F00C0007F01C00
07F03C0007FFFC0007FFFC0007F03C0007F01C0007F00C0007F00C0007F00C0307F00C03
07F0000307F0000607F0000607F0000607F0000E07F0000E07F0001E07F0003E07F001FC
FFFFFFFCFFFFFFFC20227EA125>I<FFFFFFF8FFFFFFF807F001F807F0007807F0003807
F0001807F0001C07F0001C07F0000C07F0000C07F0180C07F0180C07F0180007F0180007
F0380007F0780007FFF80007FFF80007F0780007F0380007F0180007F0180007F0180007
F0180007F0000007F0000007F0000007F0000007F0000007F0000007F0000007F00000FF
FFC000FFFFC0001E227EA123>I<0003FE0040001FFFC0C0007F00F1C001F8003FC003F0
000FC007C00007C00FC00003C01F800003C03F000001C03F000001C07F000000C07E0000
00C07E000000C0FE00000000FE00000000FE00000000FE00000000FE00000000FE000000
00FE00000000FE000FFFFC7E000FFFFC7F00001FC07F00001FC03F00001FC03F00001FC0
1F80001FC00FC0001FC007E0001FC003F0001FC001FC003FC0007F80E7C0001FFFC3C000
03FF00C026227DA12C>I<FFFF83FFFEFFFF83FFFE07F0001FC007F0001FC007F0001FC0
07F0001FC007F0001FC007F0001FC007F0001FC007F0001FC007F0001FC007F0001FC007
F0001FC007F0001FC007F0001FC007FFFFFFC007FFFFFFC007F0001FC007F0001FC007F0
001FC007F0001FC007F0001FC007F0001FC007F0001FC007F0001FC007F0001FC007F000
1FC007F0001FC007F0001FC007F0001FC007F0001FC007F0001FC0FFFF83FFFEFFFF83FF
FE27227EA12C>I<FFFF80FFFF8007F00007F00007F00007F00007F00007F00007F00007
F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007
F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F000FFFF80FF
FF8011227EA115>I<FFFF803FFCFFFF803FFC07F000038007F000070007F0000E0007F0
00180007F000300007F000E00007F001C00007F003800007F007000007F00E000007F018
000007F03C000007F0FC000007F1FE000007F3FF000007F77F000007FE3F800007F83FC0
0007F01FE00007F00FE00007F007F00007F007F80007F003FC0007F001FC0007F001FE00
07F000FF0007F0007F0007F0003F8007F0003FC007F0001FE0FFFF81FFFEFFFF81FFFE27
227EA12C>75 D<FFFFC000FFFFC00007F0000007F0000007F0000007F0000007F0000007
F0000007F0000007F0000007F0000007F0000007F0000007F0000007F0000007F0000007
F0000007F0000007F0000007F0000007F0000007F0001807F0001807F0001807F0001807
F0003807F0003807F0007007F0007007F000F007F001F007F007F0FFFFFFF0FFFFFFF01D
227EA122>I<FFF000000FFFFFF800001FFF07F800001FE006FC000037E006FC000037E0
06FC000037E0067E000067E0067E000067E0063F0000C7E0063F0000C7E0061F800187E0
061F800187E0060FC00307E0060FC00307E0060FC00307E00607E00607E00607E00607E0
0603F00C07E00603F00C07E00601F81807E00601F81807E00601F81807E00600FC3007E0
0600FC3007E006007E6007E006007E6007E006003FC007E006003FC007E006001F8007E0
06001F8007E006001F8007E006000F0007E0FFF00F00FFFFFFF00600FFFF30227EA135>
I<FFF8001FFEFFFC001FFE07FC0000C007FE0000C006FF0000C0067F8000C0063FC000C0
061FE000C0060FE000C0060FF000C00607F800C00603FC00C00601FE00C00600FE00C006
00FF00C006007F80C006003FC0C006001FE0C006000FF0C0060007F0C0060007F8C00600
03FCC0060001FEC0060000FFC00600007FC00600007FC00600003FC00600001FC0060000
0FC006000007C006000003C006000003C0FFF00001C0FFF00000C027227EA12C>I<0007
FC0000003FFF800000FC07E00003F001F80007E000FC000FC0007E001F80003F001F8000
3F003F00001F803F00001F807F00001FC07E00000FC07E00000FC0FE00000FE0FE00000F
E0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0
7E00000FC07F00001FC07F00001FC03F00001F803F80003F801F80003F000FC0007E0007
E000FC0003F001F80000FC07E000003FFF80000007FC000023227DA12A>I<FFFFFF00FF
FFFFE007F007F007F001FC07F000FC07F0007E07F0007E07F0007F07F0007F07F0007F07
F0007F07F0007F07F0007E07F0007E07F000FC07F001FC07F007F007FFFFE007FFFF0007
F0000007F0000007F0000007F0000007F0000007F0000007F0000007F0000007F0000007
F0000007F0000007F0000007F00000FFFF8000FFFF800020227EA126>I<FFFFFE0000FF
FFFFC00007F007F00007F001F80007F000FC0007F0007E0007F0007F0007F0007F0007F0
007F0007F0007F0007F0007F0007F0007F0007F0007E0007F000FC0007F001F80007F007
F00007FFFFC00007FFFF800007F00FE00007F007F00007F003F80007F001FC0007F001FC
0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC06
07F000FE0607F000FF0CFFFF803FF8FFFF800FF027227EA12A>82
D<01FE0207FF861F01FE3C007E7C001E78000E78000EF80006F80006FC0006FC0000FF00
00FFE0007FFF007FFFC03FFFF01FFFF80FFFFC03FFFE003FFE0003FE00007F00003F0000
3FC0001FC0001FC0001FE0001EE0001EF0003CFC003CFF00F8C7FFE080FF8018227DA11F
>I<7FFFFFFF807FFFFFFF807E03F80F807803F807807003F803806003F80180E003F801
C0E003F801C0C003F800C0C003F800C0C003F800C0C003F800C00003F800000003F80000
0003F800000003F800000003F800000003F800000003F800000003F800000003F8000000
03F800000003F800000003F800000003F800000003F800000003F800000003F800000003
F800000003F800000003F800000003F8000001FFFFF00001FFFFF00022227EA127>I<FF
FF803FFCFFFF803FFC07F000018007F000018007F000018007F000018007F000018007F0
00018007F000018007F000018007F000018007F000018007F000018007F000018007F000
018007F000018007F000018007F000018007F000018007F000018007F000018007F00001
8007F000018007F000018007F000018007F000018003F000030003F800030001F8000600
00FC000E00007E001C00003F80F800000FFFE0000001FF000026227EA12B>I<FFFF000F
FEFFFF000FFE07F00000C007F00000C003F800018003F800018003FC00038001FC000300
01FE00030000FE00060000FF000600007F000C00007F000C00007F801C00003F80180000
3FC03800001FC03000001FE03000000FE06000000FE060000007F0C0000007F0C0000007
F9C0000003F980000003FD80000001FF00000001FF00000000FE00000000FE00000000FE
000000007C000000007C00000000380000000038000027227FA12A>I<FFFF0FFFF00FFE
FFFF0FFFF00FFE0FF000FF0000E007F0007F0000C007F0007F0000C003F8003F80018003
F8003F80018003FC003FC0018001FC003FC0030001FC003FC0030001FE007FE0070000FE
006FE0060000FE006FE00600007F00C7F00C00007F00C7F00C00007F81C7F80C00003F81
83F81800003F8183F81800003FC301FC3800001FC301FC3000001FC301FE3000000FE600
FE6000000FE600FE6000000FFE00FFE0000007FC007FC0000007FC007FC0000003F8003F
80000003F8003F80000003F8003F80000001F0001F00000001F0001F00000001F0001F00
000000E0000E00000000E0000E000037227FA13A>I<7FFFC1FFF87FFFC1FFF803FC000C
0001FE00180001FE00380000FF003000007F806000007F80E000003FC1C000001FE18000
001FE30000000FF700000007FE00000003FC00000003FC00000001FE00000000FE000000
00FF00000000FF80000001FFC0000001BFC00000031FE00000070FF000000E0FF000000C
07F800001803FC00003803FC00007001FE00006000FF0000C000FF0001C0007F80018000
3FC0FFFC03FFFEFFFC03FFFE27227FA12A>I<FFFF000FFEFFFF000FFE07F80000C007F8
0001C003FC00018003FE00030001FE00070000FF00060000FF000C00007F801C00003FC0
1800003FC03800001FE03000000FF06000000FF0E0000007F8C0000003FD80000003FF80
000001FF00000001FE00000000FE00000000FE00000000FE00000000FE00000000FE0000
0000FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE000000
1FFFF000001FFFF00027227FA12A>I<3FFFFFE03FFFFFE03F801FC03E003FC03C003F80
38007F007000FF007000FE007001FE006003FC006003F8006007F8000007F000000FE000
001FE000001FC000003FC000007F8000007F000000FF000000FE006001FC006003FC0060
03F8006007F800E00FF000E00FE000E01FE001C01FC001C03F8003C07F8007C07F003FC0
FFFFFFC0FFFFFFC01B227DA122>I<0FFC003FFF807E07C07E03E07E01E07E01F03C01F0
0001F00001F0003FF003FDF01FC1F03F01F07E01F0FC01F0FC01F0FC01F0FC01F07E02F0
7E0CF81FF87F07E03F18167E951B>97 D<FF000000FF0000001F0000001F0000001F0000
001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0FE0
001F3FF8001FE07C001F803E001F001F001F000F801F000F801F000FC01F000FC01F000F
C01F000FC01F000FC01F000FC01F000FC01F000FC01F000F801F001F801F801F001FC03E
001EE07C001C3FF800180FC0001A237EA21F>I<00FF8007FFE00F83F01F03F03E03F07E
03F07C01E07C0000FC0000FC0000FC0000FC0000FC0000FC00007C00007E00007E00003E
00181F00300FC06007FFC000FF0015167E9519>I<0001FE000001FE0000003E0000003E
0000003E0000003E0000003E0000003E0000003E0000003E0000003E0000003E0000003E
0001FC3E0007FFBE000F81FE001F007E003E003E007E003E007C003E00FC003E00FC003E
00FC003E00FC003E00FC003E00FC003E00FC003E00FC003E007C003E007C003E003E007E
001F00FE000F83BE0007FF3FC001FC3FC01A237EA21F>I<00FE0007FF800F87C01E01E0
3E01F07C00F07C00F8FC00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C0000
7E00003E00181F00300FC07003FFC000FF0015167E951A>I<001FC0007FE000F1F001E3
F003E3F007C3F007C1E007C00007C00007C00007C00007C00007C000FFFE00FFFE0007C0
0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0
0007C00007C00007C00007C00007C0003FFC003FFC00142380A211>I<01FE0F0007FFBF
800F87C7801F03E7801E01E0003E01F0003E01F0003E01F0003E01F0003E01F0001E01E0
001F03E0000F87C0000FFF800009FE000018000000180000001C0000001FFFE0000FFFF8
0007FFFE001FFFFF003C003F0078000F80F0000780F0000780F0000780F000078078000F
003C001E001F007C000FFFF80001FFC00019217F951C>I<FF000000FF0000001F000000
1F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F000000
1F0000001F07E0001F1FF8001F307C001F403C001F803E001F803E001F003E001F003E00
1F003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E00
1F003E001F003E001F003E00FFE1FFC0FFE1FFC01A237EA21F>I<1C003E007F007F007F
003E001C000000000000000000000000000000FF00FF001F001F001F001F001F001F001F
001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B247EA310>I<FF00
0000FF0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F00
00001F0000001F0000001F0000001F00FF801F00FF801F0038001F0060001F01C0001F03
80001F0700001F0E00001F1C00001F7E00001FFF00001FCF00001F0F80001F07C0001F03
E0001F01E0001F01F0001F00F8001F007C001F003C00FFE0FFC0FFE0FFC01A237EA21E>
107 D<FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F
001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F00FFE0FF
E00B237EA210>I<FF07F007F000FF1FFC1FFC001F303E303E001F403E403E001F801F80
1F001F801F801F001F001F001F001F001F001F001F001F001F001F001F001F001F001F00
1F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F00
1F001F001F001F001F001F001F001F001F001F00FFE0FFE0FFE0FFE0FFE0FFE02B167E95
30>I<FF07E000FF1FF8001F307C001F403C001F803E001F803E001F003E001F003E001F
003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E001F
003E001F003E001F003E00FFE1FFC0FFE1FFC01A167E951F>I<00FE0007FFC00F83E01E
00F03E00F87C007C7C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007EFC007E7C
007C7C007C3E00F81F01F00F83E007FFC000FE0017167E951C>I<FF0FE000FF3FF8001F
E07C001F803E001F001F001F001F801F001F801F000FC01F000FC01F000FC01F000FC01F
000FC01F000FC01F000FC01F000FC01F001F801F001F801F803F001FC03E001FE0FC001F
3FF8001F0FC0001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F
000000FFE00000FFE000001A207E951F>I<FE1F00FE3FC01E67E01EC7E01E87E01E87E0
1F83C01F00001F00001F00001F00001F00001F00001F00001F00001F00001F00001F0000
1F00001F0000FFF000FFF00013167E9517>114 D<0FF3003FFF00781F00600700E00300
E00300F00300FC00007FE0007FF8003FFE000FFF0001FF00000F80C00780C00380E00380
E00380F00700FC0E00EFFC00C7F00011167E9516>I<0180000180000180000180000380
000380000780000780000F80003F8000FFFF00FFFF000F80000F80000F80000F80000F80
000F80000F80000F80000F80000F80000F80000F81800F81800F81800F81800F81800F83
0007C30003FE0000F80011207F9F16>I<FF01FE00FF01FE001F003E001F003E001F003E
001F003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E001F003E
001F003E001F003E001F003E001F007E001F00FE000F81BE0007FF3FC001FC3FC01A167E
951F>I<FFE01FE0FFE01FE01F8007000F8006000FC00E0007C00C0007E00C0003E01800
03E0180001F0300001F0300000F8600000F86000007CC000007CC000007FC000003F8000
003F8000001F0000001F0000000E0000000E00001B167F951E>I<FFE07FC0FFE07FC00F
801C0007C0380003E0700003F0600001F8C00000F98000007F8000003F0000001F000000
1F8000003FC0000037C0000063E00000C1F00001C0F8000380FC0007007E000E003E00FF
80FFE0FF80FFE01B167F951E>120 D<FFE01FE0FFE01FE01F8007000F8006000FC00E00
07C00C0007E00C0003E0180003E0180001F0300001F0300000F8600000F86000007CC000
007CC000007FC000003F8000003F8000001F0000001F0000000E0000000E0000000C0000
000C00000018000078180000FC380000FC300000FC60000069C000007F8000001F000000
1B207F951E>I<7FFFF07FFFF07C03E07007C0600FC0E01F80C01F00C03E00C07E0000FC
0000F80001F00003F03007E03007C0300F80701F80703F00603E00E07C03E0FFFFE0FFFF
E014167E9519>I E /Fl 82 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8700000
00000070F8F8F870051C779B18>33 D<030600078F00078F00078F00078F00078F00078F
007FFFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFF
E0FFFFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>35
D<00C00001C00001C00001C00003F0000FFC003FFE007DCF0071C700E1C380E1C780E1C7
80E1C780F1C00079C0003FC0001FE0000FF80001FC0001DE0001CF0001C70061C380F1C3
80F1C380E1C380E1C70071C70079DE003FFE001FF80007E00001C00001C00001C00000C0
0011247D9F18>I<387C7C7E3E0E0E0E1C1C38F8F0C0070E789B18>39
D<007000F001E003C007800F001E001C00380038007000700070007000E000E000E000E0
00E000E000E000E0007000700070007000380038001C001E000F00078003C001F000F000
700C24799F18>I<6000F00078003C001E000F000780038001C001C000E000E000E000E0
0070007000700070007000700070007000E000E000E000E001C001C0038007800F001E00
3C007800F00060000C247C9F18>I<01C00001C00001C00001C00041C100F1C780FDDF80
7FFF001FFC0007F00007F0001FFC007FFF00FDDF80F1C78041C10001C00001C00001C000
01C00011147D9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>44
D<7FFFC0FFFFE0FFFFE07FFFC013047E8F18>I<3078FCFC78300606778518>I<00030000
0780000780000F80000F00001F00001E00001E00003E00003C00007C0000780000780000
F80000F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F00001F
00001E00003E00003C00003C00007C0000780000F80000F00000F0000060000011247D9F
18>I<01F00007FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E000E0
E000E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C0380380
1C07001F1F000FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80FF80
FB8043800380038003800380038003800380038003800380038003800380038003800380
7FFCFFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000
E06000E00000E00000E00001C00001C00003C0000780000F00001E00003C0000780000F0
0001E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001F
FE003FFF007807807803C07801C03001C00001C0000380000380000F0003FF0003FE0003
FF000007800003C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803F
FF001FFE0003F800131C7E9B18>I<001F00003F0000770000770000E70001E70001C700
0387000787000707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF8
000700000700000700000700000700000700007FF0007FF0007FF0151C7F9B18>I<3FFF
803FFF803FFF803800003800003800003800003800003800003800003800003BF8003FFE
003FFF003C07803003C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F
803FFF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C038
0180380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F0
00E07000E07000E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>I<
E00000FFFFE0FFFFE0FFFFC0E00380E00700000F00001E00001C00003800003800007000
00F00000E00000E00001C00001C00001C000038000038000038000038000070000070000
070000070000070000070000070000131D7E9C18>I<03F0000FFC001FFE003C0F007807
80700380E001C0E001C0E001C0E001E0E001E07001E07803E03C0FE01FFFE00FFEE003F0
E00000E00001C00001C00001C0300380780780780F00783E003FFC001FF00007C000131C
7E9B18>57 D<3078FCFC783000000000000000003078FCFC78300614779318>I<183C7E
7E3C180000000000000000183C7E7E3E1E0E1C3C78F060071A789318>I<0000C00003E0
0007E0001FC0003F8000FE0001FC0007F0000FE0003F80007F0000FC0000FC00007F0000
3F80000FE00007F00001FC0000FE00003F80001FC00007E00003E00000C013187E9918>
I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FFFFE0FFFFE07FFF
C0130C7E9318>I<600000F80000FC00007F00003F80000FE00007F00001FC0000FE0000
3F80001FC00007E00007E0001FC0003F8000FE0001FC0007F0000FE0003F80007F0000FC
0000F8000060000013187E9918>I<0FF0003FFC007FFF00700F00F00380F00380600780
000F00003E00007C0001F00001E00003C00003C00003C00003C00003C000038000000000
00000000000000000000000003800007C00007C00007C000038000111C7D9B18>I<007C
0001FE0007FF000F87801E03C03C1DC0387FC070FFE071E3E071C1E0E1C1E0E380E0E380
E0E380E0E380E0E380E0E380E0E1C1C071C1C071E3C070FF80387F003C1C001E00E00F83
E007FFC001FF80007E00131C7E9B18>I<00700000F80000F80000D80000D80001DC0001
DC0001DC00018C00038E00038E00038E00038E000707000707000707000707000707000F
FF800FFF800FFF800E03801C01C01C01C01C01C07F07F0FF07F87F07F0151C7F9B18>I<
7FFC00FFFF007FFF801C03C01C01C01C00E01C00E01C00E01C00E01C01E01C01C01C07C0
1FFF801FFF001FFFC01C03C01C00E01C00F01C00701C00701C00701C00701C00F01C00E0
1C03E07FFFC0FFFF807FFE00141C7F9B18>I<00F8E003FEE007FFE00F07E01E03E03C01
E03800E07000E07000E0700000E00000E00000E00000E00000E00000E00000E00000E000
007000007000E07000E03800E03C00E01E01C00F07C007FF8003FE0000F800131C7E9B18
>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00E01C00E01C00F01C00701C
00701C00701C00701C00701C00701C00701C00701C00F01C00E01C00E01C01E01C01C01C
03C01C0F807FFF00FFFE007FF800141C7F9B18>I<FFFFF0FFFFF0FFFFF01C00701C0070
1C00701C00701C00001C00001C0E001C0E001C0E001FFE001FFE001FFE001C0E001C0E00
1C0E001C00001C00001C00381C00381C00381C00381C0038FFFFF8FFFFF8FFFFF8151C7F
9B18>I<FFFFF8FFFFF8FFFFF81C00381C00381C00381C00381C00001C00001C07001C07
001C07001FFF001FFF001FFF001C07001C07001C07001C00001C00001C00001C00001C00
001C00001C0000FFC000FFC000FFC000151C7F9B18>I<01F1C003FDC00FFFC01F0FC01C
03C03803C03801C07001C07001C0700000E00000E00000E00000E00000E00000E00FF0E0
1FF0E00FF07001C07001C07003C03803C03803C01C07C01F0FC00FFFC003FDC001F1C014
1C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C01C0
1C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C01C0
1C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001C0
0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0
0001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF00FFFF807FFF
00111C7D9B18>I<01FFC001FFC001FFC0000E00000E00000E00000E00000E00000E0000
0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0000
0E00F00E00F00E00F03C007FFC003FF0000FC000121C7D9B18>I<7F07F0FF87F87F07F0
1C03C01C07801C07001C0E001C1E001C3C001C38001C70001CF0001DF0001DF0001FB800
1FB8001F1C001E1C001C0E001C0E001C07001C07001C03801C03801C01C07F03F0FF87F8
7F03F0151C7F9B18>I<FFC000FFC000FFC0001C00001C00001C00001C00001C00001C00
001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00
E01C00E01C00E01C00E01C00E0FFFFE0FFFFE0FFFFE0131C7E9B18>I<FC01F8FE03F8FE
03F83B06E03B06E03B06E03B06E03B8EE03B8EE0398CE0398CE039DCE039DCE039DCE038
D8E038D8E038F8E03870E03870E03800E03800E03800E03800E03800E03800E0FE03F8FE
03F8FE03F8151C7F9B18>I<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C0
1CC1C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C0
1C19C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FFE003FFF
807FFFC07803C07001C0F001E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000
E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07C07C07FFF
C03FFF800FFE00131C7E9B18>I<FFFE00FFFF80FFFFC01C03C01C01E01C00E01C00701C
00701C00701C00701C00701C00E01C01E01C03C01FFFC01FFF801FFE001C00001C00001C
00001C00001C00001C00001C00001C0000FF8000FF8000FF8000141C7F9B18>I<0FFE00
3FFF807FFFC07803C07001C0F001E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0
E000E0E000E0E000E0E000E0E000E0E000E0E000E0E070E0E070E0F079E07039C0783FC0
7FFFC03FFF800FFE00000F000007800007800003C00001C00001C013227E9B18>I<7FF8
00FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF
001FFE001FFE001C0F001C07001C03801C03801C03801C03801C03801C039C1C039C1C03
9C7F01F8FF81F87F00F0161C7F9B18>I<03F1C01FFFC03FFFC07C0FC07003C0E001C0E0
01C0E001C0E000007000007800003F00001FF00007FE0000FF00000F800003C00001C000
00E00000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<
7FFFF8FFFFF8FFFFF8E07038E07038E07038E07038007000007000007000007000007000
007000007000007000007000007000007000007000007000007000007000007000007000
00700007FF0007FF0007FF00151C7F9B18>I<FF83FEFF83FEFF83FE1C00701C00701C00
701C00701C00701C00701C00701C00701C00701C00701C00701C00701C00701C00701C00
701C00701C00701C00701C00700E00E00F01E00783C003FF8001FF00007C00171C809B18
>I<FE03F8FF07F8FE03F83C01E01C01C01C01C01C01C01E03C00E03800E03800E03800E
0380070700070700070700070700038E00038E00038E00038E00018C0001DC0001DC0001
DC0000D80000F80000F800007000151C7F9B18>I<FE03F8FE03F8FE03F8700070700070
7000703800E03800E03800E03800E03800E038F8E039FCE039DCE039DCE019DCC019DCC0
19DCC0198CC0198CC01D8DC01D8DC01D8DC01D05C00D05800F07800F07800E0380151C7F
9B18>I<7F0FE07F9FE07F0FE00E07000F0700070E00078E00039C0003DC0001F80001F8
0000F80000F00000700000F00000F80001FC0001DC00039E00038E00070F000707000E07
800E03801E03C07F07F0FF07F87F07F0151C7F9B18>I<FE03F8FF07F8FE03F81C01C01E
03C00E03800F0780070700070700038E00038E0001DC0001DC0001DC0000F80000F80000
700000700000700000700000700000700000700000700000700001FC0003FE0001FC0015
1C7F9B18>I<3FFFE07FFFE07FFFE07001C07003C0700780700700000F00001E00001C00
003C0000780000700000F00001E00001C00003C0000780000700000F00001E00E01C00E0
3C00E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>I<FFF8FFF8FFF8E000E000E0
00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0
00E000E000E000E000E000E000E000E000E000FFF8FFF8FFF80D24779F18>I<FFF8FFF8
FFF800380038003800380038003800380038003800380038003800380038003800380038
0038003800380038003800380038003800380038003800380038FFF8FFF8FFF80D247F9F
18>93 D<7FFFC0FFFFE0FFFFE07FFFC013047E7F18>95 D<0FF0001FFC003FFE003C0F00
18070000038000038000FF8007FF801FFF807F0380780380E00380E00380E00380F00780
780F803FFFF81FFDF807F0F815147E9318>97 D<7E0000FE00007E00000E00000E00000E
00000E00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E00380E
00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C809B
18>I<01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E00000
E00000E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F
80003F80001F8000038000038000038000038000038003E3800FFB801FFF803C1F80380F
80700780700380E00380E00380E00380E00380E00380E00380700780700780380F803C1F
801FFFF00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F0038078070038070
0380E001C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E07C01FFF8007
FF0001F80012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C000
7FFFC0FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C000
01C00001C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007FF
F80FFFF81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001FF8
0039E0003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E000
387800F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E00000E
00000E00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E00E
00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC7FCFFE7FE7FC7FC17
1C809B18>I<038007C007C007C0038000000000000000007FC0FFC07FC001C001C001C0
01C001C001C001C001C001C001C001C001C001C001C0FFFFFFFFFFFF101D7C9C18>I<7E
0000FE00007E00000E00000E00000E00000E00000E00000E3FF00E3FF00E3FF00E07800E
0F000E1E000E3C000E78000EF0000FF8000FFC000F9C000F0E000E0F000E07000E03800E
03C07FC7F8FFC7F87FC7F8151C7F9B18>107 D<FFC000FFC000FFC00001C00001C00001
C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001
C00001C00001C00001C00001C00001C00001C00001C000FFFF80FFFF80FFFF80111C7D9B
18>I<F9C1C0FFF7F0FFFFF03E3E383C3C383C3C38383838383838383838383838383838
383838383838383838383838383838383838FE3E3EFE7E7EFE3E3E1714809318>I<7E3E
00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00
E00E00E00E00E00E00E00E00E07FC7FCFFE7FE7FC7FC1714809318>I<01F0000FFE001F
FF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0F001E07001C078
03C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807FFFC00FC1E0
0F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F00700F80E0
0FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00000E00007FC000
FFE0007FC000151E809318>I<01F38007FB801FFF803E1F80380F80700780700780E003
80E00380E00380E00380E00380E00380700780700780380F803C1F801FFF800FFB8003E3
80000380000380000380000380000380000380000380001FF0003FF8001FF0151E7E9318
>I<FF0FC0FF3FE0FF7FF007F0F007E06007C00007800007800007000007000007000007
0000070000070000070000070000070000FFFC00FFFE00FFFC0014147E9318>I<07F700
3FFF007FFF00780F00E00700E00700E007007C00007FE0001FFC0003FE00001F00600780
E00380E00380F00380F80F00FFFF00FFFC00E7F00011147D9318>I<0180000380000380
000380000380007FFFC0FFFFC0FFFFC00380000380000380000380000380000380000380
000380000380000380400380E00380E00380E001C1C001FFC000FF80003E0013197F9818
>I<7E07E0FE0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E
00E00E00E00E00E00E00E00E01E00F03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0
FF8FF87F8FF01C01C00E03800E03800E0380070700070700070700078F00038E00038E00
038E0001DC0001DC0001DC0000F80000F80000700015147F9318>I<FF07F8FF8FF8FF07
F83800E03800E03800E03800E01C01C01C71C01CF9C01CF9C01CD9C01DDDC00DDD800DDD
800DDD800D8D800F8F800F8F8007070015147F9318>I<7F8FF07F9FF07F8FF00F070007
8E00039E0001DC0001F80000F80000700000F00000F80001DC00039E00038E000707000E
07807F8FF0FF8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380
070380070700070700038700038700038E0001CE0001CE0001CC0000CC0000DC00007800
00780000780000700000700000700000F00000E00079E0007BC0007F80003F00001E0000
151E7F9318>I<0007E0001FE0007FE000780000E00000E00000E00000E00000E00000E0
0000E00000E00000E00000E00000E00001E0007FC000FF8000FF80007FC00001E00000E0
0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000007800007F
E0001FE00007E013247E9F18>123 D<7C0000FF0000FFC00003C00000E00000E00000E0
0000E00000E00000E00000E00000E00000E00000E00000E00000F000007FC0003FE0003F
E0007FC000F00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0
0000E00003C000FFC000FF00007C000013247E9F18>125 D E /Fm
55 123 df<00003FE00000E0100001803800038078000380780007003000070000000700
0000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01
C0001C01C0001C0380003803800038038000380380003807000038070000780700007007
1000700E2000700E2000700E2000600E2000E0064000E0038000E0000000C0000000C000
0001C000003180000079800000F3000000620000003C0000001D29829F1A>12
D<1C1C3E3E7E7E7E7E3A3A0202040404040808080810102020404080800F0E789F17>34
D<1C3E7E7E3A020404080810204080070E769F0E>39 D<1C3C3C3C3C0404080810202040
80060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<00000040000000C0000001800000
01800000030000000300000006000000060000000C000000180000001800000030000000
3000000060000000C0000000C0000001800000018000000300000003000000060000000C
0000000C0000001800000018000000300000003000000060000000C0000000C000000180
0000018000000300000003000000060000000C0000000C00000018000000180000003000
00003000000060000000C0000000C0000000800000001A2D7FA117>47
D<000200020006000E003C00DC031C001C0038003800380038007000700070007000E000
E000E000E001C001C001C001C003800380038003800780FFF80F1E7B9D17>49
D<0007C000182000302000E0E001C1E00181E00380C00700000F00000E00001E00001E78
001D84003E06003E07003C07007C0780780780780780780780700F00700F00F00F00F00E
00F01E00701C00601C0070380030700010C0000F8000131F7B9D17>54
D<04E0100BF0100FF0201FF0401E19C0380E80300180600300400300800600000E00000C
00001C0000180000380000700000700000F00000E00001E00001E00003C00003C00003C0
000780000780000780000F80000F00000F0000060000141F799D17>I<000F800030C000
406000803001803003003003003003003007006007804007C0C003E18003F60001FC0000
FC0001FE00033F00041F80180F80100380300380600180600180600180C00300C0030060
06006004002018001830000FC000141F7C9D17>I<001F0000718000C0C00180C00380E0
0700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03C01C07C01C0F80
0C0F8006378003C700000F00000E00000E00001C00601C00F03800F07000E0600080C000
4380003E0000131F7B9D17>I<070F1F1F0E0000000000000000000070F8F8F0E008147B
930E>I<00000200000006000000060000000E0000001E0000001E0000003F0000002F00
00004F000000CF0000008F0000010F0000010F0000020F0000020F0000040F0000080F00
00080F0000100F800010078000200780003FFF8000400780008007800080078001000780
010007800200078006000780040007801E0007C0FF807FF81D207E9F22>65
D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C003C003C003C00
3C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000F003C001E001
C001E001E001E001E001E001E003C001E003C001E003C001E003C001C0078003C0078007
8007800F0007801E000F007C00FFFFE0001E1F7D9E20>I<0000FE0200078186001C004C
0038003C0060003C00C0001C01C0001803800018070000180F0000181E0000101E000010
3C0000003C00000078000000780000007800000078000000F0000000F0000000F0000000
F0000000F00000807000008070000080700001003800010038000200180004000C001800
060020000381C00000FE00001F217A9F21>I<00FFFF80001E00E0001E0070001E003800
1E001C003C001C003C000E003C000E003C000E0078000E0078000E0078000E0078000E00
F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E0007803C0007003
C0007003C000E003C001C0078003C00780038007800E0007801C000F007000FFFFC0001F
1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004003C0004003C
0004003C000400780408007804000078040000780C0000F0180000FFF80000F0180000F0
180001E0100001E0100001E0100001E0001003C0002003C0002003C0004003C000400780
00800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<0000FC04000703
0C001C00980030007800E0007801C000380380003003800030070000300E0000301E0000
201E0000203C0000003C00000078000000780000007800000078000000F0000000F000FF
F0F0000780F0000780F0000F0070000F0070000F0070000F0070001E0038001E0018003E
001C002E000E00CC000383040000FC00001E217A9F23>71 D<00FFF1FFE0001F003E0000
1E003C00001E003C00001E003C00003C007800003C007800003C007800003C0078000078
00F000007800F000007800F000007800F00000F001E00000FFFFE00000F001E00000F001
E00001E003C00001E003C00001E003C00001E003C00003C007800003C007800003C00780
0003C007800007800F000007800F000007800F000007800F00000F801F0000FFF1FFE000
231F7D9E22>I<01FFF0001F00001E00001E00001E00003C00003C00003C00003C000078
0000780000780000780000F00000F00000F00000F00001E00001E00001E00001E00003C0
0003C00003C00003C0000780000780000780000780000F8000FFF000141F7D9E12>I<00
0FFF8000007C000000780000007800000078000000F0000000F0000000F0000000F00000
01E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000007800000
0780000007800000078000000F0000000F0000300F0000780F0000F81E0000F81E0000F0
3C0000803800004070000020E000001F80000019207D9E18>I<00FFF01FE0001F000F80
001E000E00001E000800001E001000003C002000003C004000003C010000003C02000000
7804000000780800000078100000007830000000F0F0000000F1F8000000F278000000F4
78000001E83C000001F03C000001E03C000001E01E000003C01E000003C01E000003C00F
000003C00F000007800F00000780078000078007800007800780000F8007C000FFF03FF8
00231F7D9E23>I<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000
003C0000003C00000078000000780000007800000078000000F0000000F0000000F00000
00F0000001E0000001E0000001E0000001E0008003C0010003C0010003C0030003C00200
078006000780060007800C0007801C000F007800FFFFF800191F7D9E1D>I<01FE00007F
C0001E0000FC00001E0000F80000170001780000170001780000270002F00000270004F0
0000270004F00000270008F00000470009E00000470011E00000470021E00000470021E0
0000870043C00000838043C00000838083C00000838083C0000103810780000103820780
000103820780000103840780000203840F00000203880F00000203900F00000203900F00
000401E01E00000401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0
002A1F7D9E29>I<00FF003FE0001F000F00001F00040000178004000017800400002780
08000023C008000023C008000023C008000041E010000041E010000041F010000040F010
000080F0200000807820000080782000008078200001003C400001003C400001003C4000
01001E400002001E800002001E800002000F800002000F800004000F0000040007000004
000700000C000700001C00020000FF80020000231F7D9E22>I<01FFFF80001E00E0001E
0070001E0038001E003C003C003C003C003C003C003C003C003C00780078007800780078
00F0007800E000F003C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0
000003C0000003C0000003C0000003C00000078000000780000007800000078000000F80
0000FFF000001E1F7D9E1F>80 D<0007E040001C18C0003005800060038000C0038001C0
0180018001000380010003800100038001000380000003C0000003C0000003F8000001FF
800001FFE000007FF000001FF0000001F800000078000000780000003800000038002000
3800200038002000300060007000600060006000E0007000C000E8038000C606000081F8
00001A217D9F1A>83 D<0FFFFFFC1E03C0381803C0181003C0082003C008200780086007
80084007800840078008800F0010000F0000000F0000000F0000001E0000001E0000001E
0000001E0000003C0000003C0000003C0000003C00000078000000780000007800000078
000000F0000000F0000000F0000000F0000001F000007FFF80001E1F799E21>I<3FFC0F
F807C003C00780010007800100078001000F0002000F0002000F0002000F0002001E0004
001E0004001E0004001E0004003C0008003C0008003C0008003C00080078001000780010
007800100078001000F0002000F0002000F0002000F0004000F000400070008000700100
0030020000380400000C18000007E000001D20779E22>I<FFF003FC1F0000E00F0000C0
0F0000800F0001000F0001000F0002000F0002000F0004000F0008000F80080007801000
0780300007802000078040000780400007808000078180000781000007C2000003C20000
03C4000003CC000003C8000003D0000003D0000003E0000003C0000003C0000003800000
01800000010000001E20779E22>I<00FFF07FE0001F801F00000F000C00000F80180000
0780100000078020000007C040000003C080000003C100000003E200000001E600000001
EC00000001F800000000F000000000F000000000F800000000F8000000017C000000023C
000000063C000000043E000000081E000000101E000000201F000000400F000000800F00
0001000F8000030007800006000780001F000FC000FFC03FF800231F7E9E22>88
D<020204040808101020204040404080808080B0B0F8F8F8F8F8F870700F0E749F17>92
D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E
00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97
D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38
0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F15>I<
007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F00000
F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000780003
F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389
C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E
20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E011C01
3C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C9315>
I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E00000E
0000FFE0001C00001C00001C00001C000038000038000038000038000038000070000070
0000700000700000700000E00000E00000E00000E00000E00001C00001C00001C0000180
003180007B0000F300006600003C00001629829F0E>I<001E3000713800E0F001C07003
80700780700700E00F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03801E03800E
07800E0B8006170001E700000700000700000E00000E00300E00781C00F038006070003F
C000151D7F9315>I<01E0000FE00001C00001C00001C00001C000038000038000038000
038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C03801C0380
1C0380380700380700380700380E10700E20700E20701C20700C40E00C8060070014207D
9F17>I<00C001E001E001C000000000000000000000000000000E001300230043804700
470087000E000E000E001C001C001C003840388038807080310032001C000B1F7C9E0E>
I<01E0000FE00001C00001C00001C00001C0000380000380000380000380000700000700
000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E0003870
00387000383840707080707080707080703100E03100601E0013207D9F15>107
D<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C
0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C>I<
1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C000E
00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C070803803
8071003803807100380380E10038038062007007006400300300380021147C9325>I<1C
0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C01C
01C01C03843803883803883807083803107003303001C016147C931A>I<007C0001C300
0301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780
F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C0478
1C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380
700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C00
00FF8000171D809317>I<1C1E002661002783804787804707804703008E00000E00000E
00000E00001C00001C00001C00001C000038000038000038000038000070000030000011
147C9313>114 D<00FC030206010C030C070C060C000F800FF007F803FC003E000E700E
F00CF00CE008401020601F8010147D9313>I<018001C0038003800380038007000700FF
F007000E000E000E000E001C001C001C001C003800380038003820704070407080708031
001E000C1C7C9B0F>I<0E00C01300E02301C04381C04701C04701C08703800E03800E03
800E03801C07001C07001C07001C07101C0E20180E20180E201C1E400C264007C3801414
7C9318>I<0E03801307802307C04383C04701C04700C08700800E00800E00800E00801C
01001C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315>I<
0E00C1C01300E3C02301C3E04381C1E04701C0E04701C060870380400E0380400E038040
0E0380401C0700801C0700801C0700801C0701001C0701001C0601001C0F02000C0F0400
0E13080003E1F0001B147C931E>I<0E00C01300E02301C04381C04701C04701C0870380
0E03800E03800E03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C00
07DC00001C00001C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>
121 D<01C04003E08007F1800C1F00080200000400000800001000002000004000008000
0100000200000401000802001002003E0C0063FC0041F80080E00012147D9313>I
E /Fn 16 113 df<FFFFFFC0FFFFFFC01A027C8B23>0 D<400004C0000C600018300030
1800600C00C006018003030001860000CC0000780000300000300000780000CC00018600
0303000601800C00C0180060300030600018C0000C40000416187A9623>2
D<01800180018001800180E187718E399C0FF003C003C00FF0399C718EE1870180018001
800180018010147D9417>I<03F0000FFC001FFE003FFF007FFF807FFF80FFFFC0FFFFC0
FFFFC0FFFFC0FFFFC0FFFFC07FFF807FFF803FFF001FFE000FFC0003F00012127E9317>
15 D<000000C0000003C000000F0000003C000000F0000003C00000070000001C000000
78000001E00000078000001E00000078000000E0000000780000001E0000000780000001
E0000000780000001C0000000700000003C0000000F00000003C0000000F00000003C000
0000C0000000000000000000000000000000000000000000000000000000007FFFFF80FF
FFFFC01A247C9C23>20 D<C0000000F00000003C0000000F00000003C0000000F0000000
380000000E0000000780000001E0000000780000001E0000000780000001C00000078000
001E00000078000001E00000078000000E00000038000000F0000003C000000F0000003C
00000070000000C000000000000000000000000000000000000000000000000000000000
0000007FFFFF80FFFFFFC01A247C9C23>I<003FFFC001FFFFF0071E03FC081E00FE181E
003E701E001F701E001FC01E000F001C000F001C000F003C000E003C000E003C001C0038
001C003800380078003000780040007000800070030000F01C0000F3F00000E7800000E0
000001E0000001C0000001C0000003C00000038000000380000007000000070000000E00
00000C00000020217F9E20>80 D<0000FE000007FF80001C1FC0002007E000C003F00180
01F0030000F0060000F80E0000F81C0000781C0000783800007838000078700000787000
0078F0000070F0000070F00000F0F00000E0F00000E0F80001C0F80001C07C0003807E00
83003F0707001FFC0E000FE00C0000001000000020000000C000000F00001FFC000C7FF8
001807FF0030007FF060000FFFC00000FF001E257B9F25>I<4000C000C000C000C000C0
00C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C0
00C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C0
00C000C000FFC07FC00A2E79A114>98 D<004000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C0FFC0FF
C00A2E7EA114>I<7FC0FFC0C000C000C000C000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000C000C000C000C00040000A2E79A114>I<FF
C0FFC000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C000
C000C000C000C000C000C000C000C000C000400A2E7EA114>I<000F0038006000E001C0
01C001C001C001C001C001C001C001C001C001C001C001C001C001C0038007001E00F800
1E000700038001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C0
00E000600038000F102D7DA117>I<F8001E000700038001C001C001C001C001C001C001
C001C001C001C001C001C001C001C001C000E000600038000F0038006000E001C001C001
C001C001C001C001C001C001C001C001C001C001C001C001C0038007001E00F800102D7D
A117>I<C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0
C0C0C0C0C0C0C0C0C0C0C0C0C0022D7BA10D>106 D<0000000008000000001800000000
3000000000300000000060000000006000000000C000000000C000000001800000000180
00000003000000000300000000060000000006000000000C000000000C00000000180000
000018000000003000000000300000000060000000006000000000C000060000C0001E00
0180002F000180004F000300008780030000078006000003C006000003C00C000003C00C
000001E018000001E018000000F030000000F030000000786000000078600000003CC000
00003CC00000001F800000001F800000000F000000000F00000000060000000006000000
252E7E8126>112 D E /Fo 9 115 df<07FFF80000E00E0000E0030000E0038000E00180
01C001C001C001C001C000C001C000C0038001C0038001C0038001C0038001C007000380
0700038007000300070007000E000E000E000C000E0018000E0070001C01C000FFFF0000
1A177F961D>68 D<07F007F800F000C000B8008000B80080009C0080011C0100011E0100
010E0100010E0100020702000207020002038200020382000401C4000401C4000400E400
0400E4000800780008007800080038000800380018001000FE0010001D177F961C>78
D<7C0018001800180018003000300030003000678068C070406060C060C060C060C06080
C080C08180C10046003C000B177E960F>98 D<07C00C20107020706000C000C000C00080
008000C010C02060C03F000C0E7E8D0F>I<00E2031E060E0C0E0C0C180C180C180C1018
10181018183808F00730003000306060E060C1C07F000F14808D11>103
D<0300038003000000000000000000000000001C002400460046008C000C001800180018
0031003100320032001C0009177F960C>105 D<07C00C20101020186018C018C018C018
80308030C060C0C061803E000D0E7E8D11>111 D<1C3C22462382230346030603060306
030C060C060C0C0C081A3019E018001800300030003000FC001014808D12>I<30F04918
4E384C309C001800180018003000300030003000600060000D0E7F8D10>114
D E /Fp 24 115 df<70F8F8F87005057C840D>58 D<70F0F8F878080808101010202040
050E7C840D>I<000001C00000078000001E00000078000001E00000078000000E000000
38000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003
C0000000F0000000380000000E0000000780000001E0000000780000001E000000078000
0001C01A1A7C9723>I<0000400000C00001800001800001800003000003000003000006
00000600000C00000C00000C000018000018000018000030000030000060000060000060
0000C00000C00000C0000180000180000180000300000300000600000600000600000C00
000C00000C0000180000180000300000300000300000600000600000600000C00000C000
00122D7EA117>I<E0000000780000001E0000000780000001E0000000780000001C0000
000700000003C0000000F00000003C0000000F00000003C0000003C000000F0000003C00
0000F0000003C00000070000001C00000078000001E00000078000001E00000078000000
E00000001A1A7C9723>I<000002000000060000000E0000000E0000001E0000001F0000
002F0000006F0000004F0000008F0000008F0000010F0000030F0000020F0000040F8000
040F800008078000180780001007800020078000200780007FFF80008007800080078001
0007C0010003C0020003C0040003C0040003C00C0003C03C0007C0FF003FFC1E207E9F22
>65 D<01FFFFE0001E0078001E003C001E001C001E001E003C001E003C001E003C001E00
3C001E0078003C0078003C00780078007800F000F003C000FFFF0000F007C000F000E001
E000F001E0007801E0007801E0007803C0007803C0007803C0007803C00070078000F007
8001E0078003C0078007800F001E00FFFFF0001F1F7E9E22>I<00007F00800003C0C180
000E00230000380017000070000F0000E0000F0001C0000600038000060007000006000F
000006000E000004001E000004003C000000003C00000000780000000078000000007800
0000007800000000F000000000F000000000F000000000F0000000007000002000700000
200070000020007000004000380000800038000080001C000100000E0006000007001800
0001C0600000007F80000021217F9F21>I<01FFFFE000001E003800001E000E00001E00
0700001E000700003C000380003C000380003C0001C0003C0001C000780001C000780001
C000780001C000780001C000F00003C000F00003C000F00003C000F00003C001E0000780
01E000078001E000070001E0000F0003C0000E0003C0001E0003C0001C0003C000380007
80007000078000E000078001C00007800700000F001C0000FFFFF00000221F7E9E26>I<
01FFF0001F00001E00001E00001E00003C00003C00003C00003C00007800007800007800
00780000F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C000
03C0000780000780000780000780000F8000FFF000141F7E9E14>73
D<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000003C0000003C00
000078000000780000007800000078000000F0000000F0000000F0000000F0000001E000
0001E0000001E0000001E0004003C0008003C0008003C0018003C0010007800300078003
000780060007800E000F007C00FFFFFC001A1F7E9E1F>76 D<01FF001FF8001F0003C000
1F800100001780010000178001000023C002000023C002000021E002000021E002000041
F004000040F004000040F004000040780400008078080000807C080000803C080000803C
080001001E100001001E100001000F100001000F100002000FA000020007A000020007A0
00020003E000040003C000040003C000040001C0000C0001C0001E00008000FFC0008000
251F7E9E25>78 D<01FFFFE0001E0078001E001C001E000E001E000F003C000F003C000F
003C000F003C000F0078001E0078001E0078003C0078007800F000E000F003C000FFFE00
00F0000001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C00000
078000000780000007800000078000000F800000FFF80000201F7E9E1D>80
D<0003F040000C08C0003005800060038000C0038001C001800180010003800100038001
00038001000380000003C0000003E0000003FC000001FFC00000FFF000007FF800001FF8
000001FC0000007C0000003C0000001C0000001C0020001C0020001C0020001800600038
0060003000600070007000E000E8018000C603000081FC00001A217E9F1C>83
D<0FFFFFFC1E03C0381803C0181003C0082003C008200780086007800840078008400780
08800F0010000F0000000F0000000F0000001E0000001E0000001E0000001E0000003C00
00003C0000003C0000003C00000078000000780000007800000078000000F0000000F000
0000F0000000F0000001F000007FFFC0001E1F7F9E1B>I<7FFC07FE07C000F007800040
07800040078000400F0000800F0000800F0000800F0000801E0001001E0001001E000100
1E0001003C0002003C0002003C0002003C00020078000400780004007800040078000400
F0000800F0000800F0000800F00010007000200070002000700040003800800018030000
0E0C000003F000001F207D9E1F>I<001E3000713800E0F001C0700380700780700700E0
0F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03801E03800E07800E0B80061700
01E700000700000700000E00000E00300E00781C00F038006070003FC000151D809316>
103 D<00E001E001E000C000000000000000000000000000000E00130023804380438043
808700070007000E000E001C001C001C20384038403840388019000E000B1F7E9E10>
105 D<01E0000FE00001C00001C00001C00001C000038000038000038000038000070000
0700000701E00706100E08700E10F00E20F00E20601C40001D80001E00001FC000387000
383800383800381C20703840703840703840701880E01880600F0014207E9F18>107
D<01C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C
0038003800380038007000700070007100E200E200E200E200640038000A207E9F0E>I<
007C0001C3000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0
F003C0F00780F00700700F00700E0030180018700007C00013147E9316>111
D<03C1E004621804741C08781C08701E08701E10E01E00E01E00E01E00E01E01C03C01C0
3C01C03C01C0380380780380700380E003C1C0072380071E000700000700000E00000E00
000E00000E00001C00001C0000FFC000171D819317>I<00F0400388C00705800E03801C
03803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C00F01C0070
3C00705C0030B8000F380000380000380000700000700000700000700000E00000E0000F
FC00121D7E9314>I<1C1E002621004743804787804707804703008E00000E00000E0000
0E00001C00001C00001C00001C000038000038000038000038000070000030000011147E
9315>I E /Fq 67 124 df<000FF01FC000007FF8FFF00000F81FE0780001E03F80F800
03E07F80F80007C07F00F80007C07F00F80007C03F00700007C01F00000007C01F000000
07C01F00000007C01F03FC00FFFFFFFFFC00FFFFFFFFFC0007C01F007C0007C01F007C00
07C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C00
07C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C00
07C01F007C0007C01F007C003FF8FFE3FF803FF8FFE3FF802920809F2C>14
D<387CFEFFFF7F3B030306060E0C18302008107C9F0F>39 D<0030006000C00180038007
000F000E001E001C003C003C0038007800780078007800F800F000F000F000F000F000F0
00F000F000F000F800780078007800780038003C003C001C001E000E000F000700038001
8000C0006000300C2D7CA114>I<C0006000300018001C000E000F0007000780038003C0
03C001C001E001E001E001E001F000F000F000F000F000F000F000F000F000F001F001E0
01E001E001E001C003C003C00380078007000F000E001C00180030006000C0000C2D7DA1
14>I<387CFEFEFE7C3807077C860F>46 D<0000600000E00000E00001C00001C0000380
000380000380000700000700000700000E00000E00001C00001C00001C00003800003800
00380000700000700000E00000E00000E00001C00001C000038000038000038000070000
0700000700000E00000E00001C00001C00001C0000380000380000380000700000700000
E00000E00000C00000132D7DA11A>I<01FC0007FF001F07C01E03C03E03E07C01F07C01
F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01
F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A
>I<00700000F00007F000FFF000F9F00001F00001F00001F00001F00001F00001F00001
F00001F00001F00001F00001F00001F00001F00001F00001F00001F00001F00001F00001
F00001F00001F00001F0007FFFC07FFFC0121D7D9C1A>I<03FC001FFF80381FC07C07E0
FE03F0FE03F0FE03F8FE01F87C01F83801F80003F80003F00003F00007E0000FC0000F00
001E00003C0000700000E00001C0180380180600180C00381FFFF03FFFF07FFFF0FFFFF0
FFFFF0151D7E9C1A>I<03FC000FFF801C0FC03C07E07E03F07E03F07E03F07E07F03C07
E00007E0000FC0003F8003FE0003FC00000F800007C00003E00003F00003F83803F87C03
F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C000
03C00007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383C00703C00E
03C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C000
07C00007C000FFFE00FFFE171D7F9C1A>I<1C00E01FFFE01FFFC01FFF801FFF001FFC00
1FC00018000018000018000018000019FE001FFF801E07C01803E01001F00001F00001F8
0001F87801F8FC01F8FC01F8FC01F8FC01F07803F07003E03C0FC00FFF0003FC00151D7E
9C1A>I<003F0001FFC007C0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC10
00FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01
F83C01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87F
FFF07FFFE07FFFC0E00180C00300C00300C00600000C0000180000380000300000700000
F00000F00001E00001E00003E00003E00003E00003E00007E00007E00007E00007E00007
E00003C000018000151E7D9D1A>I<01FE000FFF801E07C03801E03800F07800F07800F0
7C00F07F00F07FC1E03FE3C01FFF800FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8
F001F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>
I<01FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01
F8FC01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03
E07E03C03C0780381F001FFC0007F000151D7E9C1A>I<387CFEFEFE7C38000000000000
387CFEFEFE7C3807147C930F>I<7FFFFFFF80FFFFFFFFC0FFFFFFFFC000000000000000
000000000000000000000000000000000000000000000000000000000000000000000000
0000FFFFFFFFC0FFFFFFFFC07FFFFFFF80220F7D9229>61 D<0000E000000000E0000000
01F000000001F000000001F000000003F800000003F800000006FC00000006FC0000000E
FE0000000C7E0000000C7E000000183F000000183F000000303F800000301F800000701F
C00000600FC00000600FC00000C007E00000FFFFE00001FFFFF000018003F000038003F8
00030001F800030001F800060000FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0
231F7E9E28>65 D<FFFFFE00FFFFFFC007C007E007C003F007C001F807C001FC07C001FC
07C001FC07C001FC07C001FC07C001F807C003F807C003F007C00FE007FFFF8007FFFFC0
07C003F007C001F807C001FC07C000FC07C000FE07C000FE07C000FE07C000FE07C000FE
07C000FC07C001FC07C003F807C007F0FFFFFFE0FFFFFF001F1F7E9E25>I<0007FC0200
3FFF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E
0000067E000006FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E
0000007E0000067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000
FE01C0003FFF800007FC001F1F7D9E26>I<FFFFFE0000FFFFFFC00007E007F00007E001
F80007E000FC0007E0007E0007E0003F0007E0003F0007E0001F8007E0001F8007E0001F
8007E0001FC007E0001FC007E0001FC007E0001FC007E0001FC007E0001FC007E0001FC0
07E0001FC007E0001FC007E0001F8007E0001F8007E0001F8007E0003F0007E0003F0007
E0007E0007E000FC0007E001F80007E007F000FFFFFFC000FFFFFE0000221F7E9E28>I<
FFFFFFE0FFFFFFE007E007E007E001E007E000E007E0006007E0007007E0003007E00030
07E0303007E0303007E0300007E0700007E0F00007FFF00007FFF00007E0F00007E07000
07E0300007E0300C07E0300C07E0000C07E0001807E0001807E0001807E0003807E00078
07E000F807E003F0FFFFFFF0FFFFFFF01E1F7E9E22>I<FFFFFFE0FFFFFFE007E007E007
E001E007E000E007E0006007E0007007E0003007E0003007E0303007E0303007E0300007
E0700007E0F00007FFF00007FFF00007E0F00007E0700007E0300007E0300007E0300007
E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00000FFFF8000FF
FF80001C1F7E9E21>I<0007FC0200003FFF0E0000FE03DE0003F000FE0007E0003E000F
C0001E001F80001E003F00000E003F00000E007F000006007E000006007E00000600FE00
000000FE00000000FE00000000FE00000000FE00000000FE001FFFE0FE001FFFE07E0000
7E007E00007E007F00007E003F00007E003F00007E001F80007E000FC0007E0007E0007E
0003F000FE0000FE01FE00003FFF8E000007FC0600231F7D9E29>I<FFFF0FFFF0FFFF0F
FFF007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E
0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007FFFFFE0007FFFFFE00
07E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007
E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E0007E00FFFF0FFFF0FFFF
0FFFF0241F7E9E29>I<FFFFFFFF07E007E007E007E007E007E007E007E007E007E007E0
07E007E007E007E007E007E007E007E007E007E007E007E007E007E007E007E0FFFFFFFF
101F7E9E14>I<FFFF00FFE0FFFF00FFE007E0001E0007E000180007E000300007E00060
0007E001C00007E003800007E006000007E00C000007E018000007E030000007E0F00000
07E1F8000007E3F8000007E6FC000007EC7E000007F87F000007F03F000007E01F800007
E00FC00007E00FE00007E007E00007E003F00007E001F80007E000FC0007E000FC0007E0
007E0007E0007F00FFFF03FFF0FFFF03FFF0241F7E9E29>75 D<FFFF8000FFFF800007E0
000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0
000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E000C007E0
00C007E000C007E001C007E001C007E001C007E0038007E0038007E00F8007E01F80FFFF
FF80FFFFFF801A1F7E9E1F>I<FFE000003FF8FFF000007FF807F000007F0006F80000DF
0006F80000DF0006F80000DF00067C00019F00067C00019F00063E00031F00063E00031F
00061F00061F00061F00061F00060F800C1F00060F800C1F000607C0181F000607C0181F
000607C0181F000603E0301F000603E0301F000601F0601F000601F0601F000600F8C01F
000600F8C01F0006007D801F0006007D801F0006003F001F0006003F001F0006003F001F
0006001E001F00FFF01E03FFF8FFF00C03FFF82D1F7E9E32>I<FFE000FFF0FFF000FFF0
07F000060007F800060006FC000600067E000600063F000600063F800600061F80060006
0FC006000607E006000603F006000601F806000601FC06000600FC060006007E06000600
3F060006001F860006001FC60006000FE600060007E600060003F600060001FE00060000
FE00060000FE000600007E000600003E000600001E000600000E00FFF0000600FFF00006
00241F7E9E29>I<001FF80000FFFF0001F81F8007E007E00FC003F01F8001F81F0000F8
3F0000FC7F0000FE7E00007E7E00007EFE00007FFE00007FFE00007FFE00007FFE00007F
FE00007FFE00007FFE00007FFE00007F7E00007E7F0000FE7F0000FE3F0000FC3F8001FC
1F8001F80FC003F007E007E001F81F8000FFFF00001FF800201F7D9E27>I<FFFFFE00FF
FFFF8007E00FE007E003F007E001F807E001F807E001FC07E001FC07E001FC07E001FC07
E001FC07E001F807E001F807E003F007E00FE007FFFF8007FFFE0007E0000007E0000007
E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007
E00000FFFF0000FFFF00001E1F7E9E24>I<FFFFFC0000FFFFFF800007E00FE00007E003
F00007E003F80007E001F80007E001FC0007E001FC0007E001FC0007E001FC0007E001F8
0007E003F80007E003F00007E00FE00007FFFF800007FFFC000007E03F000007E00F8000
07E00FC00007E007C00007E007E00007E007E00007E007E00007E007F00007E007F00007
E007F00007E007F03007E003F83007E001F860FFFF00FFC0FFFF003F80241F7E9E27>82
D<03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC00007FC0
007FFE003FFF801FFFC00FFFE007FFF000FFF80007F80000FC00007C00003CC0003CC000
3CC0003CE00038E00078F80070FE01E0EFFFC081FF00161F7D9E1D>I<7FFFFFFC7FFFFF
FC7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E006C007E006C007E0
060007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0
000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00003FFFF
C003FFFFC01F1E7E9D24>I<FFFF01FFE0FFFF01FFE007E0000C0007E0000C0007E0000C
0007E0000C0007E0000C0007E0000C0007E0000C0007E0000C0007E0000C0007E0000C00
07E0000C0007E0000C0007E0000C0007E0000C0007E0000C0007E0000C0007E0000C0007
E0000C0007E0000C0007E0000C0007E0000C0007E0000C0003E000180001F000180001F0
00300000F8006000007E03C000001FFF80000003FC0000231F7E9E28>I<FFFE003FF8FF
FE003FF80FE000038007E000030007F000070003F000060003F800060001F8000C0001FC
000C0000FC00180000FC001800007E003000007E003000007F007000003F006000003F80
E000001F80C000001FC0C000000FC18000000FC180000007E300000007E300000003F600
000003F600000003FE00000001FC00000001FC00000000F800000000F800000000700000
0000700000251F7F9E28>I<FFFC0FFFC0FFE0FFFC0FFFC0FFE00FC000FC000E000FE000
FC000E0007E000FE000C0007E000FE000C0007F000FE001C0003F001BF00180003F001BF
00180001F801BF00300001F8031F80300001F8031F80300000FC071FC0600000FC060FC0
600000FE060FC0E000007E0C07E0C000007E0C07E0C000003F0C07E18000003F1803F180
00003F1803F18000001FB803FB0000001FB001FB0000001FF001FF0000000FE000FE0000
000FE000FE00000007E000FC00000007C0007C00000007C0007C00000003C00078000000
03800038000000018000300000331F7F9E36>I<FFFE07FF80FFFE07FF8007F000700003
F000E00003F800C00001FC01800000FE038000007E070000007F060000003F8C0000001F
DC0000000FF80000000FF000000007F000000003F800000003F800000003FC00000003FE
000000067E0000000E7F0000001C3F800000181FC00000300FC00000700FE00000E007F0
0001C003F800018001F800030001FC00070000FE00FFF007FFE0FFF007FFE0231F7E9E28
>I<FFFE003FF8FFFE003FF807F000038007F000070003F800060001FC000E0001FC001C
0000FE001800007F003800003F807000003F806000001FC0E000000FE1C000000FE18000
0007F380000003FF00000001FE00000001FC00000000FC00000000FC00000000FC000000
00FC00000000FC00000000FC00000000FC00000000FC00000000FC00000000FC00000000
FC0000001FFFE000001FFFE000251F7F9E28>I<3FFFFF803FFFFF803F803F003E007F00
38007E003800FC007001FC007001F8006003F0006007F0006007E000000FC000001FC000
001F8000003F0000007F0000007E000000FC000001FC018001F8018003F0018007F00180
07E003800FC003801FC003001F8007003F000F007F001F007E007F00FFFFFF00FFFFFF00
191F7D9E20>I<07FC001FFF803F07C03F03E03F01F03F01F00C01F00001F0003FF007FD
F01F81F03E01F07C01F0F801F0F801F0F801F0FC02F07E0CF03FF87E0FE03E17147F9319
>97 D<FF0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001F0000
1F00001F1FC01F7FF01FE0F81F807C1F007E1F003E1F003E1F003F1F003F1F003F1F003F
1F003F1F003F1F003E1F003E1F007C1F807C1EC1F81C7FE0181F8018207E9F1D>I<01FE
0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00
007C00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I<0007F80007F800
00F80000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F807FEF81F
83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C
00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D>I<01FE0007FF800F83C01E01E0
3E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E0018
1E00180F807007FFE000FF8015147F9318>I<003F8000FFC003E3E007C7E00787E00F87
E00F83C00F80000F80000F80000F80000F8000FFFC00FFFC000F80000F80000F80000F80
000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80
007FF8007FF80013207F9F10>I<03FC3C0FFFFE1E079E3C03DE7C03E07C03E07C03E07C
03E07C03E03C03C01E07801FFF0013FC003000003000003800003FFF801FFFF00FFFF81F
FFFC78007C70003EF0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00171E7F93
1A>I<FF0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001F0000
1F00001F0FC01F3FE01F61F01FC0F81F80F81F00F81F00F81F00F81F00F81F00F81F00F8
1F00F81F00F81F00F81F00F81F00F81F00F81F00F8FFE7FFFFE7FF18207D9F1D>I<1C00
3E007F007F007F003E001C00000000000000000000000000FF00FF001F001F001F001F00
1F001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B217EA00E>I<FF
0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001F00001F00001F
01FE1F01FE1F00F01F00C01F03801F07001F0C001F18001F7C001FFC001F9E001F0F001E
0F801E07C01E03C01E01E01E01F01E00F8FFC3FFFFC3FF18207E9F1C>107
D<FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F
001F001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B207E9F0E>I<
FE0FE03F80FE1FF07FC01E70F9C3E01E407D01F01E807E01F01F807E01F01F007C01F01F
007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F00
7C01F01F007C01F01F007C01F01F007C01F0FFE3FF8FFEFFE3FF8FFE27147D932C>I<FE
0FC0FE3FE01E61F01EC0F81E80F81F00F81F00F81F00F81F00F81F00F81F00F81F00F81F
00F81F00F81F00F81F00F81F00F81F00F8FFE7FFFFE7FF18147D931D>I<01FF0007FFC0
1F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C
7C007C3E00F83E00F81F83F007FFC001FF0017147F931A>I<FF1FC0FF7FF01FE1F81F80
FC1F007E1F007E1F003E1F003F1F003F1F003F1F003F1F003F1F003F1F003E1F007E1F00
7C1F80FC1FC1F81F7FE01F1F801F00001F00001F00001F00001F00001F00001F0000FFE0
00FFE000181D7E931D>I<FE3E00FE7F801ECFC01E8FC01E8FC01F8FC01F03001F00001F
00001F00001F00001F00001F00001F00001F00001F00001F00001F0000FFF000FFF00012
147E9316>114 D<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007
C007E007F006F81EFFFCC7F010147E9315>I<0300030003000300070007000F000F003F
00FFFCFFFC1F001F001F001F001F001F001F001F001F001F001F061F061F061F061F060F
8C07F803F00F1D7F9C14>I<FF07F8FF07F81F00F81F00F81F00F81F00F81F00F81F00F8
1F00F81F00F81F00F81F00F81F00F81F00F81F00F81F01F81F01F80F06F807FCFF03F8FF
18147D931D>I<FFE07F80FFE07F801F001C000F8018000F80180007C0300007C0300003
E0600003E0600003F0E00001F0C00001F9C00000F9800000FF8000007F0000007F000000
3E0000003E0000001C0000001C000019147F931C>I<FFC7FE1FE0FFC7FE1FE01F00F003
001F00F803000F80F806000F80F806000FC1BC0E0007C1BC0C0007C3BE0C0003E31E1800
03E31E180003F60F380001F60F300001FE0FB00000FC07E00000FC07E00000F803E00000
7803C000007803C000003001800023147F9326>I<FFC0FF00FFC0FF000F80700007C060
0007E0C00003E1800001F3800000FF0000007E0000003E0000003F0000007F8000006F80
0000C7C0000183E0000381F0000700F8000E00FC00FF81FF80FF81FF8019147F931C>I<
FFE07F80FFE07F801F001C000F8018000F80180007C0300007C0300003E0600003E06000
03F0E00001F0C00001F9C00000F9800000FF8000007F0000007F0000003E0000003E0000
001C0000001C0000001800000018000078300000FC300000FC600000C0E00000E1C00000
7F8000001E000000191D7F931C>I<3FFFE03FFFE03C07C0380F80701F80603F00603E00
607C0000F80001F80003F00003E06007C0600F80601F80E03F00C03E01C07C03C0FFFFC0
FFFFC013147F9317>I<FFFFFF80FFFFFF801902808C1A>I E /Fr
50 122 df<1C007F007F00FF80FFC0FFC07FC07FC01CC000C000C0018001800180030003
0006000C001800300020000A157BA913>39 D<000E00001E00007E0007FE00FFFE00FFFE
00F8FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE
0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE
0000FE0000FE0000FE0000FE0000FE0000FE007FFFFE7FFFFE7FFFFE17277BA622>49
D<00FF800003FFF0000FFFFC001F03FE003800FF007C007F80FE003FC0FF003FC0FF003F
E0FF001FE0FF001FE07E001FE03C003FE000003FE000003FC000003FC000007F8000007F
000000FE000000FC000001F8000003F0000003E00000078000000F0000001E0000003C00
E0007000E000E000E001C001C0038001C0070001C00FFFFFC01FFFFFC03FFFFFC07FFFFF
C0FFFFFF80FFFFFF80FFFFFF801B277DA622>I<007F800003FFF00007FFF8000F81FE00
1F00FE003F80FF003F807F803F807F803F807F801F807F800F007F800000FF000000FF00
0000FE000001FC000001F8000007F00000FFC00000FFF0000001FC000000FE0000007F00
00007F8000003FC000003FC000003FE000003FE03C003FE07E003FE0FF003FE0FF003FE0
FF003FC0FF007FC07E007F807C007F003F01FE001FFFFC0007FFF00000FF80001B277DA6
22>I<00000E0000001E0000003E0000007E000000FE000000FE000001FE000003FE0000
077E00000E7E00000E7E00001C7E0000387E0000707E0000E07E0000E07E0001C07E0003
807E0007007E000E007E000E007E001C007E0038007E0070007E00E0007E00FFFFFFF8FF
FFFFF8FFFFFFF80000FE000000FE000000FE000000FE000000FE000000FE000000FE0000
00FE00003FFFF8003FFFF8003FFFF81D277EA622>I<080003001F803F001FFFFE001FFF
FC001FFFF8001FFFF0001FFFE0001FFF80001FFE00001C0000001C0000001C0000001C00
00001C0000001C0000001C7FC0001DFFF8001F80FC001E003E0008003F0000003F800000
1FC000001FC000001FE000001FE018001FE07C001FE0FE001FE0FE001FE0FE001FE0FE00
1FC0FC001FC078003F8078003F803C007F001F01FE000FFFF80003FFF00000FF80001B27
7DA622>I<0007F800003FFE0000FFFF0001FC078003F00F8007E01FC00FC01FC01FC01F
C01F801FC03F800F803F8000007F0000007F0000007F000000FF000000FF0FC000FF3FF8
00FF707C00FFC03E00FFC03F00FF801F80FF801FC0FF001FC0FF001FE0FF001FE0FF001F
E07F001FE07F001FE07F001FE07F001FE03F001FE03F001FC01F801FC01F803F800FC03F
0007E07E0003FFFC0000FFF000003FC0001B277DA622>I<380000003E0000003FFFFFF0
3FFFFFF03FFFFFF07FFFFFE07FFFFFC07FFFFF807FFFFF0070000E0070000E0070001C00
E0003800E0007000E000E0000001C0000001C000000380000007800000070000000F0000
001F0000001E0000003E0000003E0000007E0000007C0000007C000000FC000000FC0000
00FC000000FC000001FC000001FC000001FC000001FC000001FC000001FC000001FC0000
00F80000007000001C297CA822>I<003FC00001FFF00003FFFC0007C07E000F001F001E
001F001E000F803E000F803E000F803F000F803F800F803FC00F003FF01F001FFC1E001F
FE3C000FFFF80007FFE00003FFF00001FFFC0001FFFE0007FFFF000F0FFF801E03FFC03C
01FFC07C007FE07C001FE0F8000FE0F80007E0F80003E0F80003E0F80003E0F80003C07C
0003C07C0007803F000F001FC03E000FFFFC0003FFF800007FC0001B277DA622>I<007F
800001FFF00007FFF8000FE0FC001F807E003F803F007F003F007F001F80FF001F80FF00
1FC0FF001FC0FF001FC0FF001FE0FF001FE0FF001FE0FF001FE07F001FE07F003FE03F00
3FE01F807FE00F807FE007C1DFE003FF9FE0007E1FE000001FE000001FC000001FC00000
1FC000001F803E003F807F003F807F003F007F007E007F007C003E00F8003E03F0001FFF
E0000FFF800001FE00001B277DA622>I<000003800000000007C00000000007C0000000
000FE0000000000FE0000000000FE0000000001FF0000000001FF0000000003FF8000000
003FF8000000003FF80000000073FC0000000073FC00000000F3FE00000000E1FE000000
01E1FF00000001C0FF00000001C0FF00000003C0FF80000003807F80000007807FC00000
07003FC0000007003FC000000E001FE000000E001FE000001E001FF000001C000FF00000
1FFFFFF000003FFFFFF800003FFFFFF80000780007FC0000700003FC0000F00003FE0000
E00001FE0000E00001FE0001E00001FF0001C00000FF0003C00000FF80FFFE001FFFFEFF
FE001FFFFEFFFE001FFFFE2F297EA834>65 D<FFFFFFFC0000FFFFFFFF8000FFFFFFFFC0
0003F8001FF00003F8000FF80003F80007FC0003F80003FC0003F80003FC0003F80003FE
0003F80001FE0003F80001FE0003F80001FE0003F80003FE0003F80003FC0003F80003FC
0003F80007F80003F8000FF00003F8001FE00003F800FFC00003FFFFFE000003FFFFFFE0
0003F80007F00003F80003FC0003F80001FE0003F80001FE0003F80000FF0003F80000FF
0003F80000FF8003F80000FF8003F80000FF8003F80000FF8003F80000FF8003F80000FF
8003F80000FF0003F80001FF0003F80003FE0003F80007FC0003F8001FF800FFFFFFFFF0
00FFFFFFFFC000FFFFFFFE000029297DA831>I<00003FF001800003FFFE0380000FFFFF
8780003FF007DF8000FF8001FF8001FE00007F8003FC00003F8007F000001F800FF00000
0F801FE0000007801FE0000007803FC0000007803FC0000003807FC0000003807F800000
03807F8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF800000
0000FF8000000000FF8000000000FF8000000000FF80000000007F80000000007F800000
00007FC0000003803FC0000003803FC0000003801FE0000003801FE0000007000FF00000
070007F000000E0003FC00001E0001FE00003C0000FF8000F800003FF007E000000FFFFF
C0000003FFFF000000003FF8000029297CA832>I<FFFFFFF80000FFFFFFFF8000FFFFFF
FFE00003FC001FF80003FC0007FC0003FC0001FE0003FC0000FF0003FC00007F8003FC00
003FC003FC00001FC003FC00001FE003FC00001FE003FC00000FF003FC00000FF003FC00
000FF003FC00000FF003FC00000FF803FC00000FF803FC00000FF803FC00000FF803FC00
000FF803FC00000FF803FC00000FF803FC00000FF803FC00000FF803FC00000FF803FC00
000FF003FC00000FF003FC00000FF003FC00001FE003FC00001FE003FC00001FC003FC00
003FC003FC00007F8003FC00007F0003FC0001FE0003FC0003FC0003FC001FF800FFFFFF
FFE000FFFFFFFF8000FFFFFFFC00002D297DA835>I<FFFFFFFFF0FFFFFFFFF0FFFFFFFF
F003FC000FF003FC0003F803FC0001F803FC0000F803FC00007803FC00003803FC000038
03FC00003803FC00E03C03FC00E01C03FC00E01C03FC00E01C03FC01E00003FC01E00003
FC07E00003FFFFE00003FFFFE00003FFFFE00003FC07E00003FC01E00003FC01E00003FC
00E00703FC00E00703FC00E00703FC00E00E03FC00000E03FC00000E03FC00001E03FC00
001E03FC00001C03FC00003C03FC00007C03FC0000FC03FC0001FC03FC000FFCFFFFFFFF
F8FFFFFFFFF8FFFFFFFFF828297EA82D>I<FFFFFFFFE0FFFFFFFFE0FFFFFFFFE003FC00
1FE003FC0007F003FC0001F003FC0001F003FC0000F003FC00007003FC00007003FC0000
7003FC00007803FC00E03803FC00E03803FC00E03803FC00E00003FC01E00003FC01E000
03FC07E00003FFFFE00003FFFFE00003FFFFE00003FC07E00003FC01E00003FC01E00003
FC00E00003FC00E00003FC00E00003FC00E00003FC00000003FC00000003FC00000003FC
00000003FC00000003FC00000003FC00000003FC00000003FC000000FFFFFC0000FFFFFC
0000FFFFFC000025297EA82B>I<00003FF001800003FFFE0380000FFFFF8780003FF007
DF8000FF8001FF8001FE00007F8003FC00003F8007F000001F800FF000000F801FE00000
07801FE0000007803FC0000007803FC0000003807FC0000003807F80000003807F800000
0000FF8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF800000
0000FF8000000000FF8000000000FF80007FFFFC7F80007FFFFC7F80007FFFFC7FC00000
7F803FC000007F803FC000007F801FE000007F801FE000007F800FF000007F8007F80000
7F8003FC00007F8001FE00007F8000FF8000FF80003FF003DF80000FFFFF8F800003FFFF
078000003FF801802E297CA836>I<FFFFF01FFFFEFFFFF01FFFFEFFFFF01FFFFE03FC00
007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00
007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00
007F8003FC00007F8003FC00007F8003FFFFFFFF8003FFFFFFFF8003FFFFFFFF8003FC00
007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00
007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F8003FC00
007F8003FC00007F8003FC00007F8003FC00007F8003FC00007F80FFFFF01FFFFEFFFFF0
1FFFFEFFFFF01FFFFE2F297DA836>I<FFFFFCFFFFFCFFFFFC01FE0001FE0001FE0001FE
0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE
0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE0001FE
0001FE0001FE0001FE0001FE0001FE0001FE0001FE00FFFFFCFFFFFCFFFFFC16297EA81A
>I<FFFFFC0000FFFFFC0000FFFFFC000003FC00000003FC00000003FC00000003FC0000
0003FC00000003FC00000003FC00000003FC00000003FC00000003FC00000003FC000000
03FC00000003FC00000003FC00000003FC00000003FC00000003FC00000003FC00000003
FC00000003FC00000003FC00000003FC0000E003FC0000E003FC0000E003FC0000E003FC
0001E003FC0001C003FC0001C003FC0003C003FC0003C003FC0007C003FC0007C003FC00
0FC003FC003FC003FC00FF80FFFFFFFF80FFFFFFFF80FFFFFFFF8023297EA829>76
D<FFFE0000001FFFC0FFFE0000001FFFC0FFFF0000003FFFC003FF0000003FF00003FF00
00003FF00003BF80000077F00003BF80000077F000039FC00000E7F000039FC00000E7F0
00038FE00001C7F000038FE00001C7F0000387F0000387F0000387F0000387F0000387F0
000387F0000383F8000707F0000383F8000707F0000381FC000E07F0000381FC000E07F0
000380FE001C07F0000380FE001C07F0000380FF003807F00003807F003807F00003807F
003807F00003803F807007F00003803F807007F00003801FC0E007F00003801FC0E007F0
0003800FE1C007F00003800FE1C007F00003800FE1C007F000038007F38007F000038007
F38007F000038003FF0007F000038003FF0007F000038001FE0007F000038001FE0007F0
00038000FC0007F000038000FC0007F000FFFE00FC01FFFFC0FFFE007801FFFFC0FFFE00
7801FFFFC03A297DA841>I<FFFC0000FFFEFFFE0000FFFEFFFF0000FFFE03FF80000380
03FF8000038003BFC0000380039FE0000380039FF0000380038FF80003800387F8000380
0383FC0003800381FE0003800381FF0003800380FF80038003807FC0038003803FC00380
03801FE0038003800FF0038003800FF80380038007FC0380038003FC0380038001FE0380
038000FF0380038000FF83800380007FC3800380003FE3800380001FE3800380000FF380
03800007FB8003800007FF8003800003FF8003800001FF8003800000FF80038000007F80
038000007F80038000003F80038000001F80038000000F80FFFE00000780FFFE00000380
FFFE000003802F297DA836>I<FFFFFFF800FFFFFFFF00FFFFFFFFC003FC003FE003FC00
0FF003FC0007F803FC0007FC03FC0003FC03FC0003FE03FC0003FE03FC0003FE03FC0003
FE03FC0003FE03FC0003FE03FC0003FE03FC0003FC03FC0007FC03FC0007F803FC000FF0
03FC003FE003FFFFFF8003FFFFFE0003FC00000003FC00000003FC00000003FC00000003
FC00000003FC00000003FC00000003FC00000003FC00000003FC00000003FC00000003FC
00000003FC00000003FC00000003FC00000003FC000000FFFFF00000FFFFF00000FFFFF0
000027297DA82F>80 D<FFFFFFE00000FFFFFFFE0000FFFFFFFF800003FC007FE00003FC
000FF00003FC0007F80003FC0007FC0003FC0003FC0003FC0003FE0003FC0003FE0003FC
0003FE0003FC0003FE0003FC0003FE0003FC0003FE0003FC0003FC0003FC0007F80003FC
0007F80003FC001FE00003FC007FC00003FFFFFE000003FFFFF8000003FC00FC000003FC
007F000003FC003F800003FC003F800003FC001FC00003FC001FE00003FC001FE00003FC
001FE00003FC001FE00003FC001FE00003FC001FF00003FC001FF00003FC001FF00003FC
001FF00703FC001FF80703FC000FF80703FC0007F80EFFFFF003FE1CFFFFF001FFF8FFFF
F0003FF030297DA834>82 D<007F806003FFF0E007FFF9E00F807FE01F001FE03E0007E0
7C0003E07C0001E0FC0001E0FC0001E0FC0000E0FE0000E0FE0000E0FF000000FFC00000
7FFE00007FFFE0003FFFFC001FFFFE000FFFFF8007FFFFC003FFFFC0007FFFE00007FFF0
00007FF000000FF8000007F8000003F8600001F8E00001F8E00001F8E00001F8F00001F0
F00001F0F80003F0FC0003E0FF0007C0FFE01F80F3FFFF00E1FFFE00C01FF0001D297CA8
26>I<7FFFFFFFFFC07FFFFFFFFFC07FFFFFFFFFC07F803FC03FC07E003FC007C078003F
C003C078003FC003C070003FC001C0F0003FC001E0F0003FC001E0E0003FC000E0E0003F
C000E0E0003FC000E0E0003FC000E0E0003FC000E000003FC0000000003FC0000000003F
C0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003F
C0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003F
C0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003F
C0000000003FC00000007FFFFFE000007FFFFFE000007FFFFFE0002B287EA730>I<FFFF
E07FFFF007FFF0FFFFE07FFFF007FFF0FFFFE07FFFF007FFF003FC0001FE00001C0003FC
0001FE00001C0001FE0001FF0000380001FE0000FF0000380001FE0000FF0000380000FF
0000FF8000700000FF0000FF8000700000FF8001FFC000F000007F8001FFC000E000007F
8001FFC000E000003FC003FFE001C000003FC0039FE001C000003FC0039FE001C000001F
E0070FF0038000001FE0070FF0038000001FF00F0FF8078000000FF00E07F8070000000F
F00E07F80700000007F81E07FC0E00000007F81C03FC0E00000007FC1C03FC1E00000003
FC3801FE1C00000003FC3801FE1C00000003FE7801FF3C00000001FE7000FF3800000001
FE7000FF3800000000FFE0007FF000000000FFE0007FF000000000FFE0007FF000000000
7FC0003FE0000000007FC0003FE0000000007FC0003FE0000000003F80001FC000000000
3F80001FC0000000001F00000F80000000001F00000F80000000001F00000F8000000000
0E00000700000044297FA847>87 D<03FF80000FFFF0001F01F8003F807E003F807E003F
803F001F003F800E003F8000003F8000003F8000003F80000FFF8000FFFF8007FC3F800F
E03F803F803F803F003F807F003F80FE003F80FE003F80FE003F80FE003F807E007F807F
00DF803F839FFC0FFF0FFC01FC03FC1E1B7E9A21>97 D<FFE0000000FFE0000000FFE000
00000FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE00000
000FE00000000FE00000000FE00000000FE00000000FE00000000FE1FE00000FE7FF8000
0FFE07E0000FF801F0000FF000F8000FE000FC000FE000FE000FE0007F000FE0007F000F
E0007F000FE0007F800FE0007F800FE0007F800FE0007F800FE0007F800FE0007F800FE0
007F800FE0007F000FE0007F000FE0007F000FE000FE000FE000FC000FF001F8000FF803
F0000F9E0FE0000F0FFF80000E01FC0000212A7EA926>I<003FF80000FFFE0003F01F00
07E03F800FC03F801F803F803F801F007F000E007F0000007F000000FF000000FF000000
FF000000FF000000FF000000FF000000FF0000007F0000007F0000007F8000003F8001C0
1F8001C00FC0038007E0070003F01E0000FFFC00003FE0001A1B7E9A1F>I<00003FF800
00003FF80000003FF800000003F800000003F800000003F800000003F800000003F80000
0003F800000003F800000003F800000003F800000003F800000003F800000003F800001F
E3F80000FFFBF80003F83FF80007E00FF8000FC007F8001F8003F8003F8003F8007F0003
F8007F0003F8007F0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F8
00FF0003F800FF0003F8007F0003F8007F0003F8007F0003F8003F8003F8001F8003F800
0F8007F80007C00FF80003F03FFF8000FFF3FF80003FC3FF80212A7EA926>I<003FE000
01FFF80003F07E0007C01F000F801F801F800F803F800FC07F000FC07F0007C07F0007E0
FF0007E0FF0007E0FFFFFFE0FFFFFFE0FF000000FF000000FF0000007F0000007F000000
7F0000003F8000E01F8000E00FC001C007E0038003F81F0000FFFE00001FF0001B1B7E9A
20>I<0007F0003FFC00FE3E01F87F03F87F03F07F07F07F07F03E07F00007F00007F000
07F00007F00007F00007F000FFFFC0FFFFC0FFFFC007F00007F00007F00007F00007F000
07F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F000
07F00007F00007F00007F0007FFF807FFF807FFF80182A7EA915>I<00FF81F003FFE7F8
0FC1FE7C1F80FC7C1F007C383F007E107F007F007F007F007F007F007F007F007F007F00
7F007F003F007E001F007C001F80FC000FC1F8001FFFE00018FF80003800000038000000
3C0000003E0000003FFFF8001FFFFF001FFFFF800FFFFFC007FFFFE01FFFFFF03C0007F0
7C0001F8F80000F8F80000F8F80000F8F80000F87C0001F03C0001E01F0007C00FC01F80
03FFFE00007FF0001E287E9A22>I<FFE0000000FFE0000000FFE00000000FE00000000F
E00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE0
0000000FE00000000FE00000000FE00000000FE07F00000FE1FFC0000FE787E0000FEE03
F0000FF803F0000FF803F8000FF003F8000FF003F8000FE003F8000FE003F8000FE003F8
000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F800
0FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F800FFFE3FFF80FF
FE3FFF80FFFE3FFF80212A7DA926>I<07000F801FC03FE03FE03FE01FC00F8007000000
000000000000000000000000FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00FE0
0FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE0FFFEFFFEFFFE0F2B7DAA14>
I<FFE00000FFE00000FFE000000FE000000FE000000FE000000FE000000FE000000FE000
000FE000000FE000000FE000000FE000000FE000000FE000000FE01FFC0FE01FFC0FE01F
FC0FE007800FE00F000FE01E000FE03C000FE078000FE0E0000FE3C0000FE7C0000FEFE0
000FFFE0000FFFF0000FF3F8000FE3F8000FC1FC000FC0FE000FC07F000FC07F000FC03F
800FC01FC00FC00FC00FC00FE0FFFC3FFEFFFC3FFEFFFC3FFE1F2A7EA924>107
D<FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00F
E00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00F
E00FE00FE00FE0FFFEFFFEFFFE0F2A7DA914>I<FFC07F800FF000FFC1FFE03FFC00FFC3
83F0707E000FC603F8C07F000FCC01F9803F000FD801FF003F800FF001FE003F800FF001
FE003F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC
003F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC00
3F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC003F800FE001FC003F
800FE001FC003F80FFFE1FFFC3FFF8FFFE1FFFC3FFF8FFFE1FFFC3FFF8351B7D9A3A>I<
FFC07F0000FFC1FFC000FFC787E0000FCE03F0000FD803F0000FD803F8000FF003F8000F
F003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE0
03F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003
F8000FE003F8000FE003F800FFFE3FFF80FFFE3FFF80FFFE3FFF80211B7D9A26>I<003F
E00001FFFC0003F07E000FC01F801F800FC03F800FE03F0007E07F0007F07F0007F07F00
07F0FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F8FF0007F87F00
07F07F0007F03F800FE03F800FE01F800FC00FC01F8007F07F0001FFFC00003FE0001D1B
7E9A22>I<FFE1FE0000FFE7FF8000FFFE07E0000FF803F0000FF001F8000FE000FC000F
E000FE000FE000FF000FE0007F000FE0007F000FE0007F800FE0007F800FE0007F800FE0
007F800FE0007F800FE0007F800FE0007F800FE0007F000FE000FF000FE000FF000FE000
FE000FE001FC000FF001F8000FF803F0000FFE0FE0000FEFFF80000FE1FC00000FE00000
000FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE0000000
0FE0000000FFFE000000FFFE000000FFFE00000021277E9A26>I<FFC1F0FFC7FCFFCE3E
0FD87F0FD87F0FF07F0FF03E0FF01C0FE0000FE0000FE0000FE0000FE0000FE0000FE000
0FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE000FFFF00FFFF00FFFF00
181B7E9A1C>114 D<03FE300FFFF01E03F03800F0700070F00070F00070F80070FE0000
FFE0007FFE007FFF803FFFE01FFFF007FFF800FFF80007FC6000FCE0007CE0003CF0003C
F00038F80038FC0070FF01E0F7FFC0C1FF00161B7E9A1B>I<0070000070000070000070
0000F00000F00000F00001F00003F00003F00007F0001FFFF0FFFFF0FFFFF007F00007F0
0007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F0
3807F03807F03807F03807F03807F03803F03803F87001F86000FFC0001F8015267FA51B
>I<FFE03FF800FFE03FF800FFE03FF8000FE003F8000FE003F8000FE003F8000FE003F8
000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F800
0FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000FE003F8000F
E007F80007E007F80007E00FF80003F03BFF8001FFF3FF80003FC3FF80211B7D9A26>I<
FFFE03FF80FFFE03FF80FFFE03FF800FF000780007F000700007F800F00003F800E00003
FC01E00001FC01C00001FC01C00000FE03800000FE03800000FF078000007F070000007F
8F0000003F8E0000003FCE0000001FDC0000001FDC0000000FF80000000FF80000000FF8
00000007F000000007F000000003E000000003E000000001C00000211B7F9A24>I<FFFC
3FFC0FFEFFFC3FFC0FFEFFFC3FFC0FFE0FE007E000E007F003F001C007F003F001C007F8
07F803C003F807F8038003F807F8038001FC0EFC070001FC0EFC070001FC1EFE0F0000FE
1C7E0E0000FE1C7E0E0000FF383F1E00007F383F1C00007F783F1C00003FF01FB800003F
F01FB800003FE00FF800001FE00FF000001FE00FF000000FC007E000000FC007E000000F
C007E00000078003C00000078003C0002F1B7F9A32>I<FFFC0FFF00FFFC0FFF00FFFC0F
FF0007F003C00003F803800001FC07800000FE0F000000FF1E0000007F3C0000003FF800
00001FF00000001FF00000000FF000000007F000000007F80000000FFC0000001FFE0000
001EFE0000003C7F000000783F800000F01FC00001E01FE00003C00FE00003C007F000FF
F01FFF80FFF01FFF80FFF01FFF80211B7F9A24>I<FFFE03FF80FFFE03FF80FFFE03FF80
0FF000780007F000700007F800F00003F800E00003FC01E00001FC01C00001FC01C00000
FE03800000FE03800000FF078000007F070000007F8F0000003F8E0000003FCE0000001F
DC0000001FDC0000000FF80000000FF80000000FF800000007F000000007F000000003E0
00000003E000000001C000000001C000000003800000000380000038078000007C070000
00FE0F000000FE0E000000FE1C000000FE3C0000007C780000003FE00000000F80000000
21277F9A24>I E /Fs 87 127 df<001F83E000F06E3001C078780380F8780300F03007
007000070070000700700007007000070070000700700007007000FFFFFF800700700007
007000070070000700700007007000070070000700700007007000070070000700700007
007000070070000700700007007000070070000700700007007000070070003FE3FF001D
20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700
00070000070000FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700
E00700E00700E00700E00700E00700E00700E00700E00700E00700E03FC3FC1620809F19
>I<003FE000E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E007
00E0FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007
00E00700E00700E00700E00700E00700E00700E00700E03FE7FC1620809F19>I<001F81
F80000F04F040001C07C06000380F80F000300F00F000700F00F00070070000007007000
000700700000070070000007007000000700700000FFFFFFFF0007007007000700700700
070070070007007007000700700700070070070007007007000700700700070070070007
007007000700700700070070070007007007000700700700070070070007007007000700
70070007007007003FE3FE3FE02320809F26>I<7038F87CFC7EFC7E743A040204020402
0804080410081008201040200F0E7F9F17>34 D<00018030000001803000000180300000
0180300000030060000003006000000300600000030060000003006000000600C0000006
00C000000600C000000600C000000600C000000C0180007FFFFFFF00FFFFFFFF80001803
00000018030000003006000000300600000030060000003006000000600C000000600C00
00FFFFFFFF807FFFFFFF0000C01800000180300000018030000001803000000180300000
0180300000030060000003006000000300600000030060000003006000000600C0000006
00C000000600C0000021297E9F26>I<70F8FCFC74040404080810102040060E7C9F0D>
39 D<0040008001000300060004000C0018001800380030003000700060006000600060
00E000E000E000E000E000E000E000E000E000E000E000E0006000600060006000700030
0030003800180018000C000400060003000100008000400A2E7BA112>I<800040002000
3000180008000C00060006000700030003000380018001800180018001C001C001C001C0
01C001C001C001C001C001C001C001C00180018001800180038003000300070006000600
0C000800180030002000400080000A2E7EA112>I<01800180018001800180E187718E39
9C0FF003C003C00FF0399C718EE1870180018001800180018010147DA117>I<00030000
000300000003000000030000000300000003000000030000000300000003000000030000
0003000000030000000300000003000000030000FFFFFFFCFFFFFFFC0003000000030000
000300000003000000030000000300000003000000030000000300000003000000030000
000300000003000000030000000300001E207E9A23>I<70F0F8F8780808081010102020
40050E7C840D>I<FFF0FFF00C02808A0F>I<70F8F8F87005057C840D>I<0000400000C0
000180000180000180000300000300000300000600000600000C00000C00000C00001800
00180000180000300000300000600000600000600000C00000C00000C000018000018000
0180000300000300000600000600000600000C00000C00000C0000180000180000300000
300000300000600000600000600000C00000C00000122D7EA117>I<03F0000E1C001C0E
00180600380700700380700380700380700380F003C0F003C0F003C0F003C0F003C0F003
C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07003807003807003807807803807
001806001C0E000E1C0003F000121F7E9D17>I<008003800F80F3800380038003800380
038003800380038003800380038003800380038003800380038003800380038003800380
0380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E00200700400780800780F007C0
F803C0F803C0F803C02007C00007C0000780000780000F00000E00001C00003800007000
00600000C0000180000300000600400C00401800401000803FFF807FFF80FFFF80121E7E
9D17>I<03F0000C1C00100E00200F00780F80780780780780380F80000F80000F00000F
00001E00001C0000700007F000003C00000E00000F000007800007800007C02007C0F807
C0F807C0F807C0F00780400780400F00200E00183C0007F000121F7E9D17>I<00060000
0600000E00000E00001E00002E00002E00004E00008E00008E00010E00020E00020E0004
0E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0000E00000E00000E0000
0E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE001FFC001FF8001FE000
10000010000010000010000010000010000011F000161C00180E00100700100780000380
0003800003C00003C00003C07003C0F003C0F003C0E00380400380400700200600100C00
08380007E000121F7E9D17>I<007C000182000701000E03800C07801807803803003800
00780000700000700000F1F000F21C00F40600F80700F80380F80380F003C0F003C0F003
C0F003C0F003C07003C07003C07003803803803807001807000C0E00061C0001F000121F
7E9D17>I<4000007FFFE07FFFC07FFFC040008080010080010080020000040000040000
080000100000100000200000200000600000600000E00000C00001C00001C00001C00001
C00003C00003C00003C00003C00003C00003C00003C000018000131F7E9D17>I<03F000
0C0C001006003003002001806001806001806001807001807803003E03003F06001FC800
0FF00003F80007FC000C7E00103F00300F806007806001C0C001C0C000C0C000C0C000C0
C000806001802001001002000C0C0003F000121F7E9D17>I<03F0000E18001C0C003806
00380700700700700380F00380F00380F003C0F003C0F003C0F003C0F003C07007C07007
C03807C0180BC00E13C003E3C0000380000380000380000700300700780600780E00700C
002018001070000FC000121F7E9D17>I<70F8F8F8700000000000000000000070F8F8F8
7005147C930D>I<70F8F8F8700000000000000000000070F0F8F8780808081010102020
40051D7C930D>I<7FFFFFF8FFFFFFFC0000000000000000000000000000000000000000
000000000000000000000000FFFFFFFC7FFFFFF81E0C7E9023>61
D<0FE0103C601E400EE00FF00FF00F600F001E001C00380070006000C000800080010001
00010001000100010000000000000000000000038007C007C007C0038010207E9F15>63
D<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0
000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C
0000C03E0000801E0000801E0001FFFF0001000F0001000F000200078002000780020007
80040003C0040003C00C0003C01E0003E0FF801FFE1F207F9F22>65
D<FFFFE0000F0078000F001E000F001E000F000F000F000F800F000F800F000F800F000F
800F000F800F000F000F001F000F001E000F007C000FFFF0000F007C000F001F000F000F
800F0007C00F0003C00F0003E00F0003E00F0003E00F0003E00F0003E00F0003C00F0007
C00F0007800F000F000F003E00FFFFF0001B1F7E9E20>I<000FE01000381C3000E00270
03C00170078000F00F0000701E0000701E0000303C0000303C0000107C00001078000010
F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000
780000007C0000103C0000103C0000101E0000201E0000200F0000200780004003C00080
00E0030000380C00000FF0001C217E9F21>I<FFFFF80007801E0007800780078003C007
8001E0078000F00780007007800078078000780780003C0780003C0780003C0780003E07
80003E0780003E0780003E0780003E0780003E0780003E0780003E0780003C0780003C07
80007C0780007807800078078000F0078001E0078003C00780078007801E00FFFFF8001F
1F7F9E23>I<FFFFFF800F000F800F0003800F0001800F0000800F0000C00F0000400F00
00400F0000400F0040400F0040000F0040000F00C0000F01C0000FFFC0000F01C0000F00
C0000F0040000F0040000F0040000F0000200F0000200F0000200F0000400F0000400F00
00400F0000C00F0001C00F0003800F000F80FFFFFF801B1F7E9E1F>I<FFFFFF80078007
80078001800780018007800080078000C007800040078000400780004007800040078020
000780200007802000078060000780E00007FFE0000780E0000780600007802000078020
000780200007800000078000000780000007800000078000000780000007800000078000
0007C00000FFFE00001A1F7F9E1E>I<000FE01000381C3000E0027003C00170078000F0
0F0000701E0000701E0000303C0000303C0000107C00001078000010F8000000F8000000
F8000000F8000000F8000000F8000000F8000000F8003FFEF80001F0780000F07C0000F0
3C0000F03C0000F01E0000F01E0000F00F0000F0078000F003C0017000E0023000380C10
000FF0001F217E9F24>I<FFF07FF80F0007800F0007800F0007800F0007800F0007800F
0007800F0007800F0007800F0007800F0007800F0007800F0007800F0007800FFFFF800F
0007800F0007800F0007800F0007800F0007800F0007800F0007800F0007800F0007800F
0007800F0007800F0007800F0007800F0007800F000780FFF07FF81D1F7E9E22>I<FFF0
0F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F00
0F000F000F000F000F000F000F000F000F000F000F00FFF00C1F7E9E10>I<07FFC0003E
00001E00001E00001E00001E00001E00001E00001E00001E00001E00001E00001E00001E
00001E00001E00001E00001E00001E00001E00001E00001E00001E00201E00F81E00F81E
00F81E00F01C00403C006038001070000FC00012207F9E17>I<FFF007FC0F0003E00F00
01800F0001000F0002000F0004000F0008000F0010000F0020000F0040000F0080000F01
00000F0300000F0780000F0F80000F13C0000F21E0000F41E0000F80F0000F0078000F00
78000F003C000F001E000F001E000F000F000F0007800F0007800F0003C00F0003E00F00
03F0FFF01FFE1F1F7E9E23>I<FFF8000F80000F00000F00000F00000F00000F00000F00
000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00
000F00020F00020F00020F00020F00060F00040F00040F000C0F001C0F007CFFFFFC171F
7E9E1C>I<FF800007FE07800007C007800007C005C0000BC005C0000BC004E00013C004
E00013C004E00013C004700023C004700023C004380043C004380043C004380043C0041C
0083C0041C0083C0040E0103C0040E0103C0040E0103C004070203C004070203C0040702
03C004038403C004038403C00401C803C00401C803C00401C803C00400F003C00400F003
C004006003C01F006003C0FFE0607FFE271F7F9E2A>I<FF000FF80F8003E00F8000800B
C0008009E0008009E0008008F000800878008008780080083C0080081E0080081E008008
0F008008078080080780800803C0800801E0800801E0800800F080080078800800788008
003C8008001E8008001E8008000F80080007800800078008000380080001803E000180FF
8000801D1F7E9E22>I<001FE0000070380001C00E0003800700070003800F0003C01E00
01E03C0000F03C0000F07C0000F87C0000F878000078F800007CF800007CF800007CF800
007CF800007CF800007CF800007CF800007CF800007C780000787C0000F87C0000F83C00
00F03E0001F01E0001E00F0003C0070003800380070001E01E0000703800001FE0001E21
7E9F23>I<FFFFE0000F007C000F001E000F000F000F0007800F0007800F0007C00F0007
C00F0007C00F0007C00F0007800F0007800F000F000F001E000F007C000FFFE0000F0000
000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000
000F0000000F0000000F0000000F000000FFF000001A1F7E9E1F>I<001FE00000703800
01C00E0003800700070003800F0003C01E0001E03E0001F03C0000F07C0000F87C0000F8
78000078F800007CF800007CF800007CF800007CF800007CF800007CF800007CF800007C
F800007C780000787C0000F87C0000F83C0000F03E0781F01E0841E00F1023C007102380
0390170001D01E0000783804001FF80400001C0400000C0C00000E1C00000FF800000FF8
000007F8000007F0000001E01E297E9F23>I<FFFF80000F00F0000F003C000F001E000F
000F000F000F000F000F800F000F800F000F800F000F800F000F000F000F000F001E000F
003C000F00F0000FFF80000F01C0000F0070000F0070000F0038000F003C000F003C000F
003C000F003E000F003E000F003E000F003E040F003F040F001F040F000F08FFF0078800
0001F01E207E9E21>I<03F0400C0CC01803C03001C06000C06000C0E000C0E00040E000
40E00040F00000F800007C00007F80003FF8001FFF0007FF8000FFC0001FE00003E00001
E00000F0000070800070800070800070800070C00060C000E0E000C0F80180C6030081FC
0014217E9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010
800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000
000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000
000F0000000F0000000F0000000F0000001F800003FFFC001C1F7E9E21>I<FFF00FF80F
0003E00F0000800F0000800F0000800F0000800F0000800F0000800F0000800F0000800F
0000800F0000800F0000800F0000800F0000800F0000800F0000800F0000800F0000800F
0000800F0000800F0000800F0000800F0000800700010007800100038001000380020001
C0040000E0080000383000000FC0001D207E9E22>I<FFF003FE1F8000F80F0000600F00
002007800040078000400780004003C0008003C0008003E0018001E0010001E0010000F0
020000F0020000F802000078040000780400003C0800003C0800003C0800001E1000001E
1000001F1000000F2000000F20000007C0000007C0000007C00000038000000380000003
8000000100001F207F9E22>I<FFF07FF81FF01F000FC007C00F00078001800F00078001
000F0007C001000F8007C00300078007C00200078009E0020007C009E0020003C009E004
0003C019F0040003C010F0040001E010F0080001E010F0080001E02078080000F0207810
0000F02078100000F0403C10000078403C20000078403C20000078801E2000007C801E60
00003C801E4000003D000F4000003F000F4000001F000F8000001F000F8000001E000780
00000E00070000000E00070000000C000300000004000200002C207F9E2F>I<7FF81FF8
0FE007C007C0030003C0020003E0060001F0040000F0080000F8180000781000003C2000
003E6000001E4000000F8000000F8000000780000003C0000007E0000005E0000008F000
0018F8000010780000207C0000603E0000401E0000801F0001000F8001000780020007C0
060003C01F0007E0FFC01FFE1F1F7F9E22>I<FFF001FF1F8000780F8000600780006007
C0004003C0008003E0008001F0010000F0010000F80200007C0600003C0400003E080000
1E0800001F1000000FB0000007A0000007E0000003C0000003C0000003C0000003C00000
03C0000003C0000003C0000003C0000003C0000003C0000003C0000003C000003FFC0020
1F7F9E22>I<7FFFFE7E003C78003C7000786000F86000F04001E04003E04003C0400780
000780000F00001E00001E00003C00007C0000780000F00001F00001E00203C00203C002
0780020F80060F00061E00043E000C3C000C78003C7800FCFFFFFC171F7E9E1C>I<FFFF
C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0
C0C0C0C0C0FFFF082D7CA10D>I<080410082010201040204020804080408040B85CFC7E
FC7E7C3E381C0F0E7A9F17>I<FFFF030303030303030303030303030303030303030303
0303030303030303030303030303030303030303FFFF082D80A10D>I<08102020404080
8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00
000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720
0F83C014147E9317>I<1C0000FC00001C00001C00001C00001C00001C00001C00001C00
001C00001C00001C00001C7C001D87001E01801E00C01C00E01C00701C00701C00781C00
781C00781C00781C00781C00781C00701C00F01C00E01E00C01A0180198700107C001520
7E9F19>I<01FC000706001C0F00380F00380600780000700000F00000F00000F00000F0
0000F00000F000007000007800003800803800801C010007060001F80011147F9314>I<
0001C0000FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0
01F1C0070DC00C03C01801C03801C07801C07001C0F001C0F001C0F001C0F001C0F001C0
F001C07001C07001C03801C01803C00C03C0070DC001F1F815207F9F19>I<03F0000E1C
001C0E00380700380700700700700380F00380F00380FFFF80F00000F00000F000007000
007000003800803800801C010007060001F80011147F9314>I<007C01C6030F070F0E06
0E000E000E000E000E000E000E00FFF00E000E000E000E000E000E000E000E000E000E00
0E000E000E000E000E000E000E000E007FE01020809F0E>I<0000E003E3300E3C301C1C
30380E00780F00780F00780F00780F00780F00380E001C1C001E380033E0002000002000
003000003000003FFE001FFF801FFFC03001E0600070C00030C00030C00030C000306000
603000C01C038003FC00141F7F9417>I<1C0000FC00001C00001C00001C00001C00001C
00001C00001C00001C00001C00001C00001C7C001C86001D03001E03801E03801C03801C
03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C
0380FF8FF014207E9F19>I<38007C007C007C0038000000000000000000000000001C00
FC001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00
FF80091F7F9E0C>I<00E001F001F001F000E0000000000000000000000000007007F000
F00070007000700070007000700070007000700070007000700070007000700070007000
700070007000706070F060F0C061803F000C28829E0E>I<1C0000FC00001C00001C0000
1C00001C00001C00001C00001C00001C00001C00001C00001C1FE01C07801C06001C0400
1C08001C10001C20001C60001CE0001DF0001E70001C38001C3C001C1C001C0E001C0F00
1C07001C07801C07C0FF9FF014207E9F18>I<1C00FC001C001C001C001C001C001C001C
001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C
001C001C001C001C00FF8009207F9F0C>I<1C3E03E000FCC30C30001D039038001E01E0
1C001E01E01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C
001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C00
1C01C01C00FF8FF8FF8021147E9326>I<1C7C00FC86001D03001E03801E03801C03801C
03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C
0380FF8FF014147E9319>I<01F800070E001C03803801C03801C07000E07000E0F000F0
F000F0F000F0F000F0F000F0F000F07000E07000E03801C03801C01C0380070E0001F800
14147F9317>I<1C7C00FD87001E01801E01C01C00E01C00F01C00701C00781C00781C00
781C00781C00781C00781C00701C00F01C00E01E01C01E03801D87001C7C001C00001C00
001C00001C00001C00001C00001C00001C0000FF8000151D7E9319>I<01F040070CC00E
02C01C03C03801C07801C07001C0F001C0F001C0F001C0F001C0F001C0F001C07001C078
01C03801C01C03C00C05C00709C001F1C00001C00001C00001C00001C00001C00001C000
01C00001C0000FF8151D7F9318>I<1CF0FD181E3C1E3C1E181C001C001C001C001C001C
001C001C001C001C001C001C001C001C00FFC00E147E9312>I<0FC830386018C008C008
C008E0007C003FE01FF007F8003C800E8006C006C006C004E00CD81887E00F147F9312>
I<020002000200060006000E000E003E00FFF80E000E000E000E000E000E000E000E000E
000E000E000E040E040E040E040E040708030801F00E1C7F9B12>I<1C0380FC1F801C03
801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03
801C03801C07800C0780061B8003E3F014147E9319>I<FF83F83E00E01C00C00E00800E
00800E008007010007010007830003820003820001C40001C40001E40000E80000E80000
700000700000700000200015147F9318>I<FF9FE1FC3C0780701C0300601C0380200E03
80400E0380400E03C0400704C0800704E0800704E08003886100038871000388710001D0
320001D03A0001D03E0000E01C0000E01C0000601800004008001E147F9321>I<FF87F8
1E03C00E01800E030007020003840001C80001D80000F00000700000780000F800009C00
010E00020E000607000403800C03C03C03E0FE07FC16147F9318>I<FF83F83E00E01C00
C00E00800E00800E008007010007010007830003820003820001C40001C40001E40000E8
0000E800007000007000007000002000002000004000004000004000F08000F08000F100
006200003C0000151D7F9318>I<7FFF700E600E401C40384078407000E001E001C00380
078007010E011E011C0338027006700EFFFE10147F9314>I<FFFFFC1601808C17>I<1C04
3F0843F080E00E047C9D17>126 D E /Ft 18 122 df<70F8F8F8700505798414>46
D<01E007F00E38181C38FC71FC731E771EEE0EEE0EEE0EEE0EEE0EEE0EEE0E771C731871
F038E018060E1E07F801F00F177E9614>64 D<1FC0007FF000707800201800001C00001C
0007FC001FFC003C1C00701C00E01C00E01C00E01C00707C003FFF800F8F8011107E8F14
>97 D<03F80FFE1C0E380470006000E000E000E000E00060007000380E1C1E0FFC03F00F
107E8F14>99 D<007E00007E00000E00000E00000E00000E00000E0007CE000FFE001C3E
00301E00700E00E00E00E00E00E00E00E00E00E00E00E00E00700E00301E00383E001FEF
C007CFC012177F9614>I<07E00FF01C38301C700CE00EE00EFFFEFFFEE0006000700038
0E1C1E0FFC03F00F107E8F14>I<07CF001FFF80383B80301800701C00701C00701C0030
18003838003FF00037C0007000007000003FF8001FFC003FFE00700F00E00380E00380E0
0380E003807007003C1E001FFC0007F00011197F8F14>103 D<FC0000FC00001C00001C
00001C00001C00001C00001C78001DFE001F86001E07001C07001C07001C07001C07001C
07001C07001C07001C07001C07001C0700FF8FE0FF8FE01317809614>I<FC0000FC0000
1C00001C00001C00001C00001C00001CFF801CFF801C1C001C38001C70001CE0001DC000
1FE0001FE0001E70001C38001C3C001C1C001C0E00FF1F80FF1F801117809614>107
D<FF80FF8003800380038003800380038003800380038003800380038003800380038003
80038003800380FFFEFFFE0F177E9614>I<FC7800FDFE001F86001E07001C07001C0700
1C07001C07001C07001C07001C07001C07001C07001C0700FF8FE0FF8FE01310808F14>
110 D<07C01FF03C78701C701CE00EE00EE00EE00EE00EE00E701C783C3C781FF007C00F
107E8F14>I<FE1F00FE7F800EE3800F81000F00000F00000E00000E00000E00000E0000
0E00000E00000E00000E0000FFF000FFF00011107F8F14>114 D<0FD83FF86038C038C0
38F0007F803FF007F8001C6006E006F006F81CFFF8CFE00F107E8F14>I<06000E000E00
0E000E007FFCFFFC0E000E000E000E000E000E000E000E000E0E0E0E0E0E0E1C07F801F0
0F157F9414>I<FC3F00FC3F001C07001C07001C07001C07001C07001C07001C07001C07
001C07001C07001C07001C1F000FFFE003E7E01310808F14>I<FE3F80FE3F8070070038
0E00380E00380E0039CE0039CE0039CE001B6C001A6C001A6C001A6C001E7C001E7C000E
380011107F8F14>119 D<FE3F80FE3F801C0E001C1C001C1C000E1C000E38000E380007
380007300007300003700003700003E00001E00001E00001C00001C00001C00003800073
80007700007E00003C000011187F8F14>121 D E /Fu 49 122 df<60F0F07010101020
204040040B7D830B>44 D<FFC0FFC00A0280880D>I<60F0F06004047D830B>I<07E01C38
381C300C700E60066006E007E007E007E007E007E007E007E007E007E00760066006700E
300C381C1C3807E010187F9713>48 D<03000700FF000700070007000700070007000700
07000700070007000700070007000700070007000700070007007FF80D187D9713>I<07
E01838201C601E700E201E001E001C001C0038007007E00038001C000E000F000FE00FE0
0FC00F400E601C183807E010187F9713>51 D<001800180038007800F800B80138023802
38043808381838103820384038C038FFFF00380038003800380038003803FF10187F9713
>I<30183FF03FE03F80200020002000200020002FC03060203000380018001C001C401C
E01CE01C80184038403030E00F800E187E9713>I<01F807040C06180E300E3000700060
00E000E3E0E418E80CF00EE006E007E007E007600760077006300E180C0C3807E010187F
9713>I<40007FFF7FFE7FFE400480088010801000200040004000800180010003000300
0700060006000E000E000E000E000E00040010197E9813>I<07E01818300C2006600660
067006780C3E181F3007C003E00CF8307C601E600FC007C003C003C00360022004181807
E010187F9713>I<07E01C303018700C600EE006E006E007E007E0076007700F30171827
07C700070006000E000C700C7018603030600F8010187F9713>I<000C0000000C000000
0C0000001E0000001E0000002F0000002700000027000000438000004380000043800000
81C0000081C0000181E0000100E0000100E00003FFF00002007000020070000400380004
0038000400380008001C0008001C003C001E00FF00FFC01A1A7F991D>65
D<003F0201C0C603002E0E001E1C000E1C0006380006780002700002700002F00000F000
00F00000F00000F00000F000007000027000027800023800041C00041C00080E00080300
3001C0C0003F00171A7E991C>67 D<FFFF80000E00E0000E0070000E0038000E001C000E
000E000E000E000E0007000E0007000E0007800E0007800E0007800E0007800E0007800E
0007800E0007800E0007800E0007000E0007000E000F000E000E000E001C000E001C000E
0078000E00E000FFFF8000191A7F991D>I<FFFFF80E00380E00180E00080E000C0E0004
0E00040E00040E01000E01000E01000E03000FFF000E03000E01000E01000E01000E0000
0E00000E00000E00000E00000E00000E00000E0000FFE000161A7F9919>70
D<FFE01FC00E000F000E000C000E0008000E0010000E0020000E0040000E0180000E0200
000E0400000E0C00000E1C00000E2E00000E4700000E8380000F0380000E01C0000E00E0
000E00E0000E0070000E0038000E0038000E001C000E001E000E001F00FFE07FC01A1A7F
991E>75 D<FFE0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E
00000E00000E00000E00000E00000E00000E00000E00080E00080E00080E00180E00100E
00300E00700E00F0FFFFF0151A7F9918>I<FF0000FF0F0000F00F0000F00B8001700B80
017009C0027009C0027009C0027008E0047008E004700870087008700870087008700838
10700838107008381070081C2070081C2070080E4070080E4070080E4070080780700807
8070080300701C030070FF8307FF201A7F9923>I<FE007FC00F000E000F0004000B8004
0009C0040009C0040008E00400087004000870040008380400081C0400081C0400080E04
000807040008038400080384000801C4000800E4000800E4000800740008003C0008003C
0008001C0008000C001C000C00FF8004001A1A7F991D>I<007F000001C1C00007007000
0E0038001C001C003C001E0038000E0078000F0070000700F0000780F0000780F0000780
F0000780F0000780F0000780F0000780F000078078000F0078000F0038000E003C001E00
1C001C000E0038000700700001C1C000007F0000191A7E991E>I<FFFF800E01E00E0070
0E00780E00380E003C0E003C0E003C0E003C0E00380E00780E00700E01E00FFF800E0000
0E00000E00000E00000E00000E00000E00000E00000E00000E00000E0000FFE000161A7F
991A>I<FFFF00000E01C0000E0070000E0078000E003C000E003C000E003C000E003C00
0E003C000E0078000E0070000E01C0000FFF00000E0380000E00C0000E00E0000E007000
0E0070000E0070000E0078000E0078000E0078000E0078400E003C400E001C80FFE00F00
1A1A7F991C>82 D<0FC21836200E6006C006C002C002C002E00070007E003FE01FF803FC
007E000E00070003800380038003C002C006E004D81887E0101A7E9915>I<7FFFFF0070
1C0700401C0100401C0100C01C0180801C0080801C0080801C0080001C0000001C000000
1C0000001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C000000
1C0000001C0000001C0000001C0000001C0000001C000003FFE000191A7F991C>I<FFE0
7FC00E000E000E0004000E0004000E0004000E0004000E0004000E0004000E0004000E00
04000E0004000E0004000E0004000E0004000E0004000E0004000E0004000E0004000E00
04000E0004000600080007000800030010000180200000E0C000003F00001A1A7F991D>
I<1FC000387000383800101C00001C00001C0003FC001E1C00381C00701C00E01C00E01C
80E01C80E03C80705F801F8F0011107F8F13>97 D<FC00001C00001C00001C00001C0000
1C00001C00001C00001C00001C00001CFC001D07001E03801C01C01C00C01C00E01C00E0
1C00E01C00E01C00E01C00E01C01C01C01801E030019060010F800131A809915>I<07F8
1C1C381C70087000E000E000E000E000E000E0007000700438081C1807E00E107F8F11>
I<003F0000070000070000070000070000070000070000070000070000070003E7000C17
00180F00300700700700E00700E00700E00700E00700E00700E00700600700700700380F
001C370007C7E0131A7F9915>I<07C01C3030187018600CE00CFFFCE000E000E000E000
6000700438081C1807E00E107F8F11>I<01F007180E381C101C001C001C001C001C001C
00FFC01C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF800D1A80
990C>I<0FCF001871803030007038007038007038007038003030001860002FC0006000
006000007000003FF0003FFC001FFE00600F00C00300C00300C00300C00300600600381C
0007E00011187F8F13>I<FC00001C00001C00001C00001C00001C00001C00001C00001C
00001C00001C7C001D87001E03801E03801C03801C03801C03801C03801C03801C03801C
03801C03801C03801C03801C0380FF9FF0141A809915>I<183C3C18000000000000FC1C
1C1C1C1C1C1C1C1C1C1C1C1C1CFF081A80990A>I<FC00001C00001C00001C00001C0000
1C00001C00001C00001C00001C00001C1FC01C0F001C0C001C18001C20001C40001CE000
1DE0001E70001C78001C38001C1C001C1E001C0F001C0F80FF9FE0131A809914>107
D<FC001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C
001C001C001C001C001C001C001C00FF80091A80990A>I<FC7C1F001D8E63801E0781C0
1E0781C01C0701C01C0701C01C0701C01C0701C01C0701C01C0701C01C0701C01C0701C0
1C0701C01C0701C01C0701C0FF9FE7F81D107F8F20>I<FC7C001D87001E03801E03801C
03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C0380FF9FF014
10808F15>I<07E01C38300C700E6006E007E007E007E007E007E0076006700E381C1C38
07E010107F8F13>I<FCFC001D07001E03801C01C01C01C01C00E01C00E01C00E01C00E0
1C00E01C00E01C01C01C01801E03001D06001CF8001C00001C00001C00001C00001C0000
1C0000FF80001317808F15>I<FC781D9C1E1C1E081C001C001C001C001C001C001C001C
001C001C001C00FF800E10808F0F>114 D<1F2060E04020C020C020F0007F003FC01FE0
00F080708030C030C020F0408F800C107F8F0F>I<0800080008000800180018003800FF
C038003800380038003800380038003800382038203820382018201C4007800B177F960F
>I<FC1F801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C
03801C07800C07800E0B8003F3F01410808F15>I<FF0F803C07001C06001C04001C0400
0E08000E080007100007100007100003A00003A00001C00001C00001C00000800011107F
8F14>I<FE7F1F80381C07003C1C06001C0C04001C0E04000E1608000E1708000E170800
072310000723900007A3900003C1A00003C1E0000180C0000180C0000180C00019107F8F
1C>I<FE3F803C1E001C08000E10000F300007600003C00001C00001E00003E000027000
043800083800181C00381E00FC3FC012107F8F14>I<FF0F803C07001C06001C04001C04
000E08000E080007100007100007100003A00003A00001C00001C00001C0000080000080
00010000010000E10000E20000E4000078000011177F8F14>I E
/Fv 3 123 df<0C000C00CCC0EDC07F800C007F80EDC0CCC00C000C000A0B7D8B10>3
D<1818181818FFFF18181818181818181818181818181808167D900E>121
D<1818181818FF18181818180018181818FFFF1818181808167D900E>I
E /Fw 51 122 df<003F07E00001C09C18000380F018000701F03C000E01E03C000E00E0
18000E00E000000E00E000000E00E000000E00E000000E00E00000FFFFFFFC000E00E01C
000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C00
0E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E
00E01C007FC7FCFF80211D809C23>14 D<00800100020006000C000C0018001800300030
0030006000600060006000E000E000E000E000E000E000E000E000E000E000E000E00060
00600060006000300030003000180018000C000C000600020001000080092A7C9E10>40
D<8000400020003000180018000C000C0006000600060003000300030003000380038003
8003800380038003800380038003800380038003000300030003000600060006000C000C
00180018003000200040008000092A7E9E10>I<00060000000600000006000000060000
000600000006000000060000000600000006000000060000000600000006000000060000
FFFFFFE0FFFFFFE000060000000600000006000000060000000600000006000000060000
0006000000060000000600000006000000060000000600001B1C7E9720>43
D<60F0F0701010101020204080040C7C830C>I<FFE0FFE00B0280890E>I<60F0F0600404
7C830C>I<03C00C301818300C300C700E60066006E007E007E007E007E007E007E007E0
07E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15>48
D<010007003F00C700070007000700070007000700070007000700070007000700070007
00070007000700070007000700070007000700FFF80D1C7C9B15>I<07C01830201C400C
400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C0018003000601
0C01180110023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E00
1C001C00180030006007C00030001C001C000E000F000F700FF80FF80FF80FF00E401C20
1C183007C0101D7E9B15>I<000C00000C00001C00003C00003C00005C0000DC00009C00
011C00031C00021C00041C000C1C00081C00101C00301C00201C00401C00C01C00FFFFC0
001C00001C00001C00001C00001C00001C00001C0001FFC0121C7F9B15>I<300C3FF83F
F03FC020002000200020002000200023E02C303018301C200E000E000F000F000F600FF0
0FF00FF00F800E401E401C2038187007C0101D7E9B15>I<00F0030C04040C0E181E301E
300C700070006000E3E0E430E818F00CF00EE006E007E007E007E007E007600760077006
300E300C18180C3003E0101D7E9B15>I<4000007FFF807FFF007FFF0040020080040080
040080080000100000100000200000600000400000C00000C00001C00001800001800003
8000038000038000038000078000078000078000078000078000078000030000111D7E9B
15>I<000600000006000000060000000F0000000F0000000F0000001780000017800000
37C0000023C0000023C0000043E0000041E0000041E0000080F0000080F0000080F00001
0078000100780001FFF80002003C0002003C0002003C0004001E0004001E000C001F000C
000F001E001F00FF00FFF01C1D7F9C1F>65 D<FFFFC0000F0078000F003C000F001E000F
000E000F000F000F000F000F000F000F000F000F000E000F001E000F003C000F00F8000F
FFF0000F003C000F001E000F000F000F0007000F0007800F0007800F0007800F0007800F
0007800F000F000F000F000F001E000F007800FFFFE000191C7F9B1D>I<001F808000E0
618001801980070007800E0003801C0003801C0001803800018078000080780000807000
0080F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00000007000
00807800008078000080380000801C0001001C0001000E000200070004000180080000E0
3000001FC000191E7E9C1E>I<FFFFC0000F00F0000F003C000F000E000F0007000F0007
000F0003800F0003C00F0001C00F0001C00F0001E00F0001E00F0001E00F0001E00F0001
E00F0001E00F0001E00F0001E00F0001C00F0001C00F0003C00F0003800F0007800F0007
000F000E000F001C000F007000FFFFC0001B1C7E9B20>I<FFF00F000F000F000F000F00
0F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F00
0F000F000F00FFF00C1C7F9B0F>73 D<FFF01FE00F0007800F0006000F0004000F000800
0F0010000F0020000F0040000F0080000F0100000F0200000F0600000F0F00000F178000
0F2780000F43C0000F81E0000F01E0000F00F0000F0078000F0078000F003C000F003E00
0F001E000F000F000F000F800F000FC0FFF03FF01C1C7F9B20>75
D<FFF8000F80000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00
000F00000F00000F00000F00000F00000F00000F00080F00080F00080F00180F00180F00
100F00300F00700F01F0FFFFF0151C7E9B1A>I<FF8000FF800F8000F8000F8000F8000B
C00178000BC00178000BC001780009E002780009E002780008F004780008F004780008F0
047800087808780008780878000878087800083C107800083C107800083C107800081E20
7800081E207800081E207800080F407800080F4078000807807800080780780008078078
0008030078001C03007800FF8307FF80211C7E9B26>I<FFFF800F00E00F00780F003C0F
001C0F001E0F001E0F001E0F001E0F001E0F001C0F003C0F00780F00E00FFF800F00000F
00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000FFF00017
1C7E9B1C>80 D<07E0801C1980300580300380600180E00180E00080E00080E00080F000
00F800007C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001
C08001C08001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>83
D<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020800F0020800F00
20000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00
00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000001F80
0003FFFC001B1C7F9B1E>I<1FC000307000783800781C00301C00001C00001C0001FC00
0F1C00381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115>
97 D<FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0000
1C7C001D87001E01801C00C01C00E01C00601C00701C00701C00701C00701C00701C0070
1C00601C00E01C00C01E018019070010FC00141D7F9C17>I<03F80C0C181E301E700C60
00E000E000E000E000E000E00060007002300218040C1803E00F127F9112>I<001F8000
038000038000038000038000038000038000038000038000038000038003F3800E0B8018
0780300380700380600380E00380E00380E00380E00380E00380E0038060038070038030
03801807800E1B8003E3F0141D7F9C17>I<07E00C301818300C700E6006E006FFFEE000
E000E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E
000E000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E
000E000E000E000E007FE00F1D809C0D>I<00038007C4C01C78C0383880301800701C00
701C00701C00701C003018003838001C700027C0002000002000003000003FF8001FFF00
1FFF802003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215>
I<FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C7C
001C87001D03001E03801C03801C03801C03801C03801C03801C03801C03801C03801C03
801C03801C03801C03801C0380FF9FF0141D7F9C17>I<18003C003C0018000000000000
000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C
001C001C001C00FF80091D7F9C0C>I<00C001E001E000C0000000000000000000000000
000007E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0
00E000E000E000E060E0F0C0F1C061803E000B25839C0D>I<FC00001C00001C00001C00
001C00001C00001C00001C00001C00001C00001C00001C0FE01C07801C06001C0C001C10
001C20001C40001CE0001DE0001E70001C78001C38001C3C001C1C001C0E001C0F001C0F
80FF9FE0131D7F9C16>I<FC001C001C001C001C001C001C001C001C001C001C001C001C
001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F
9C0C>I<FC7E07E0001C838838001D019018001E01E01C001C01C01C001C01C01C001C01
C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C01C001C01C0
1C001C01C01C001C01C01C001C01C01C00FF8FF8FF8021127F9124>I<FC7C001C87001D
03001E03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C
03801C03801C0380FF9FF014127F9117>I<03F0000E1C00180600300300700380600180
E001C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F000
12127F9115>I<FC7C001D87001E01801C01C01C00E01C00E01C00701C00701C00701C00
701C00701C00701C00E01C00E01C01C01E01801D07001CFC001C00001C00001C00001C00
001C00001C00001C0000FF8000141A7F9117>I<03E0800E198018058038078070038070
0380E00380E00380E00380E00380E00380E003807003807003803807801807800E1B8003
E380000380000380000380000380000380000380000380001FF0141A7F9116>I<FCE01D
381E781E781C301C001C001C001C001C001C001C001C001C001C001C001C00FFC00D127F
9110>I<1F9020704030C010C010E010F8007F803FE00FF000F880388018C018C018E010
D0608FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C
001C001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>I<FC1F
801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03801C03
801C03801C07800C07800E1B8003E3F014127F9117>I<FF07E03C03801C01001C01000E
02000E02000F060007040007040003880003880003C80001D00001D00000E00000E00000
E00000400013127F9116>I<FF3FC7E03C0703C01C0701801C0701001C0B01000E0B8200
0E0B82000E1182000711C4000711C4000720C40003A0E80003A0E80003C0680001C07000
01C0700001803000008020001B127F911E>I<FF0FE01E07001C06000E04000708000390
0003B00001E00000E00000F00001F000013800021C00041C000C0E001807003C0780FE1F
F014127F9116>I<FF07E03C03801C01001C01000E02000E02000F060007040007040003
880003880003C80001D00001D00000E00000E00000E000004000004000008000008000F0
8000F10000F30000660000380000131A7F9116>I E /Fx 7 117
df<00038000000380000007C0000007C0000007C000000FE000000FE000001FF000001B
F000003BF8000031F8000031F8000060FC000060FC0000E0FE0000C07E0000C07E000180
3F0001FFFF0003FFFF8003001F8007001FC006000FC006000FC00C0007E00C0007E0FF80
3FFEFF803FFE1F1C7E9B24>65 D<0FF8001C1E003E0F803E07803E07C01C07C00007C000
7FC007E7C01F07C03C07C07C07C0F807C0F807C0F807C0780BC03E13F80FE1F815127F91
17>97 D<FF0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001F00
001F3F801FE1E01F80701F00781F003C1F003C1F003E1F003E1F003E1F003E1F003E1F00
3E1F003C1F003C1F00781F80701EC1E01C3F00171D7F9C1B>I<03FC000E0E001C1F003C
1F00781F00780E00F80000F80000F80000F80000F80000F800007800007801803C01801C
03000E0E0003F80011127E9115>I<FE3E00FE47001E8F801E8F801E8F801F07001F0000
1F00001F00001F00001F00001F00001F00001F00001F00001F0000FFE000FFE00011127F
9114>114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01CC01CE01C
E018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFFFC1F001F
001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F9913>I
E /Fy 3 123 df<FFFFF0FFFFF014027D881B>0 D<060006000600060006000600060006
00FFF0FFF006000600060006000600060006000600060006000600060006000600060006
000600060006000C1D7E9611>121 D<060006000600060006000600FFF0FFF006000600
06000600060006000000060006000600060006000600FFF0FFF006000600060006000600
06000C1D7E9611>I E /Fz 31 127 df<70F8FCFC7404040404080810102040060F7C84
0E>44 D<70F8F8F87005057C840E>46 D<01F000071C000C060018030038038038038070
01C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F0
01E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C07803C03803803803801C
07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003
800380038003800380038003800380038003800380038003800380038003800380038003
80038003800380038007C0FFFE0F217CA018>I<03F0000C1C001007002007804003C040
03C08003E0F003E0F801E0F801E0F801E02003E00003E00003C00003C000078000070000
0E00001C0000180000300000600000C00001800001000002002004002008002018006030
00403FFFC07FFFC0FFFFC013217EA018>I<03F8000C1E00100F002007804007C07807C0
7803C07807C03807C0000780000780000700000F00000C0000380003F000001C00000F00
0007800007800003C00003C00003E02003E07003E0F803E0F803E0F003C04003C0400780
200780100F000C1C0003F00013227EA018>I<1000801E07001FFF001FFE001FF80017E0
0010000010000010000010000010000010000011F800120C001C07001803801003800001
C00001C00001E00001E00001E00001E07001E0F001E0F001E0E001C08001C04003C04003
802007001006000C1C0003F00013227EA018>53 D<4000006000007FFFE07FFFC07FFFC0
400080C00100800100800200800200000400000800000800001000002000002000006000
00400000C00000C00001C00001C000018000038000038000038000038000078000078000
07800007800007800007800007800003000013237DA118>55 D<01F000060C000C060018
0700380380700380700380F001C0F001C0F001C0F001E0F001E0F001E0F001E0F001E070
01E07003E03803E01805E00C05E00619E003E1E00001C00001C00001C000038000038030
0380780700780600700C002018001030000FC00013227EA018>57
D<0007E0100038183000E0063001C00170038000F0070000F00E0000701E0000701C0000
303C0000303C0000307C0000107800001078000010F8000000F8000000F8000000F80000
00F8000000F8000000F8000000F800000078000000780000107C0000103C0000103C0000
101C0000201E0000200E000040070000400380008001C0010000E0020000381C000007E0
001C247DA223>67 D<FFFFF0000F801E000780070007800380078001C0078000E0078000
F007800078078000780780007C0780003C0780003C0780003C0780003E0780003E078000
3E0780003E0780003E0780003E0780003E0780003E0780003E0780003C0780003C078000
7C0780007807800078078000F0078000E0078001E0078003C0078007000F801E00FFFFF0
001F227EA125>I<03FFE0001F00000F00000F00000F00000F00000F00000F00000F0000
0F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000
0F00000F00000F00000F00000F00700F00F80F00F80F00F80E00F01E00401C0020380018
700007C00013237EA119>74 D<FFFC001F80000F00000F00000F00000F00000F00000F00
000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00
000F00000F00000F00010F00010F00010F00010F00030F00030F00020F00060F00060F00
1E1F007EFFFFFE18227DA11E>76 D<FFC00003FF0FC00003F007C00003E005E00005E005
E00005E004F00009E004F00009E004F00009E004780011E004780011E004780011E0043C
0021E0043C0021E0043C0021E0041E0041E0041E0041E0040F0081E0040F0081E0040F00
81E004078101E004078101E004078101E00403C201E00403C201E00401E401E00401E401
E00401E401E00400F801E00400F801E00400F801E004007001E00E007001E01F007003F0
FFE0203FFF28227EA12D>I<FFFFE000000F803C000007800E00000780078000078007C0
00078003C000078003E000078003E000078003E000078003E000078003E000078003C000
078007C000078007800007800E000007803C000007FFE000000780700000078038000007
801C000007801E000007800E000007800F000007800F000007800F000007800F00000780
0F800007800F800007800F800007800F808007800FC080078007C0800FC003C100FFFC01
E2000000007C0021237EA124>82 D<FFFC07FF0FC000F807800070078000200780002007
800020078000200780002007800020078000200780002007800020078000200780002007
800020078000200780002007800020078000200780002007800020078000200780002007
80002007800020078000200380004003C0004003C0004001C0008000E000800060010000
300600001C08000003F00020237EA125>85 D<FFF03FFC03FE1F8007E000F80F0003C000
700F0003C000200F0001E00020078001E00040078001E00040078003F0004003C002F000
8003C002F0008003C002F0008003E00478018001E00478010001E00478010001E0083C01
0000F0083C020000F0083C020000F0101E02000078101E04000078101E04000078200F04
00003C200F0800003C200F0800003C600F8800001E40079000001E40079000001E4007D0
00001F8003F000000F8003E000000F8003E000000F0001E00000070001C00000070001C0
0000060000C0000002000080002F237FA132>87 D<1FE000303800780C00780E00300700
00070000070000070000FF0007C7001E07003C0700780700700700F00708F00708F00708
F00F087817083C23900FC1E015157E9418>97 D<01FE000703000C07801C078038030078
0000700000F00000F00000F00000F00000F00000F00000F000007000007800403800401C
00800C010007060001F80012157E9416>99 D<01FC000707000C03801C01C03801C07801
E07000E0F000E0FFFFE0F00000F00000F00000F00000F000007000007800203800201C00
400E008007030000FC0013157F9416>101 D<00007003F1980E1E181C0E183807003807
007807807807807807807807803807003807001C0E001E1C0033F0002000002000003000
003800003FFE001FFFC00FFFE03000F0600030C00018C00018C00018C000186000306000
303800E00E038003FE0015217F9518>103 D<0E0000FE00001E00000E00000E00000E00
000E00000E00000E00000E00000E00000E00000E00000E00000E1F800E60C00E80E00F00
700F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00
700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C003E003E003E001C00000000
000000000000000000000000000E007E001E000E000E000E000E000E000E000E000E000E
000E000E000E000E000E000E000E000E00FFC00A227FA10E>I<0E0000FE00001E00000E
00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E
01F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E
07000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>107
D<0E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E
000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B
237FA20E>I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E0070
0E00700E00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F
941B>110 D<01FC000707000C01801800C03800E0700070700070F00078F00078F00078
F00078F00078F00078F000787000707800F03800E01C01C00E038007070001FC0015157F
9418>I<0E3CFE461E8F0F0F0F060F000E000E000E000E000E000E000E000E000E000E00
0E000E000E000F00FFF010157F9413>114 D<02000200020002000600060006000E001E
003E00FFFC0E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E
040E040708030801F00E1F7F9E13>116 D<FF80FE1E00781E00300E00200E0020070040
0700400780C003808003808001C10001C10000E20000E20000E200007400007400003800
003800003800001000001000002000002000002000004000F04000F08000F18000430000
3C0000171F7F941A>121 D<0E023F04238841F880E00F057CA018>126
D E /FA 1 4 df<00C00000C00000C00000C00000C000E0C1C0F0C3C038C7000EDC0003
F00000C00003F0000EDC0038C700F0C3C0E0C1C000C00000C00000C00000C00000C00012
157D9619>3 D E /FB 29 119 df<78FCFCFEFE7A020202020204040408081020204007
147AB112>39 D<78FCFCFCFC7806067A8512>46 D<00080000380000780001F8003FF800
FE7800C07800007800007800007800007800007800007800007800007800007800007800
007800007800007800007800007800007800007800007800007800007800007800007800
007800007800007800007800007800007800007800007800007800007800007800007800
00780000780000780000FC007FFFF87FFFF8152F7AAE21>49 D<00000C0000000C000000
1C0000001C0000003C0000007C0000007C000000BC0000013C0000013C0000023C000006
3C0000043C0000083C0000183C0000103C0000203C0000603C0000403C0000803C000180
3C0001003C0002003C0002003C0004003C0008003C0008003C0010003C0020003C002000
3C0040003C00C0003C00FFFFFFF8FFFFFFF800003C0000003C0000003C0000003C000000
3C0000003C0000003C0000003C0000003C0000003C0000007E00000FFFF0000FFFF01D2F
7EAE21>52 D<003F800000FFE00003E0F000078038000F001C001E000E001C000F003C00
07007C0007807800078078000780F80007C0F80003C0F80003C0F80003C0F80003C0F800
03E0F80003E0F80003E0F80003E0780003E0780007E0780007E03C0007E03C0007E01C00
0BE00E0013E0070013E0038023E001C0C3E0003F03C0000003C0000003C0000007C00000
0780000007800000078000000F0000000F0038000E007C001C007C003C00780038007000
70003000E0001C03C0000FFF000001FC00001B307DAE21>57 D<00000300000000000300
000000000300000000000780000000000780000000000FC0000000000FC0000000000FC0
0000000017E00000000013E00000000013E00000000023F00000000021F00000000021F0
0000000040F80000000040F80000000040F800000000807C00000000807C00000001007E
00000001003E00000001003E00000002003F00000002001F00000002001F00000004000F
80000004000F80000004000F800000080007C00000080007C00000180007E000001FFFFF
E000001FFFFFE00000200003F00000200001F00000200001F00000400000F80000400000
F80000400000F800008000007C00008000007C00018000007E00010000003E0001000000
3E00030000003F00030000001F00070000001F001F8000003F80FFE00001FFFCFFE00001
FFFC2E327EB132>65 D<FFFFFFE000FFFFFFFC0007E0003E0003E0000F8003E00007C003
E00003E003E00001E003E00001F003E00000F003E00000F803E00000F803E00000F803E0
0000F803E00000F803E00000F803E00000F003E00001F003E00001F003E00003E003E000
07C003E0000F8003E0001F0003E0007C0003FFFFF80003E0001F0003E000078003E00003
E003E00001F003E00000F803E000007803E000007C03E000007C03E000003E03E000003E
03E000003E03E000003E03E000003E03E000003E03E000003E03E000007C03E000007C03
E000007C03E00000F803E00001F003E00003E003E00007C007E0003F80FFFFFFFE00FFFF
FFF80027317CB02F>I<00003FE0010001FFF8030007F01E03001F800307003E00008700
7800004F00F000002F01E000001F03C000000F078000000F0F800000070F000000071F00
0000031E000000033E000000033C000000017C000000017C000000017C00000001780000
0000F800000000F800000000F800000000F800000000F800000000F800000000F8000000
00F800000000F800000000F800000000F80000000078000000007C000000007C00000001
7C000000013C000000013E000000011E000000011F000000020F000000020F8000000607
8000000403C000000801E000000800F00000100078000020003E0000C0001F8003800007
F00F000001FFFC0000003FE00028337CB130>I<00001FE000800000FFFC01800007F00F
0180000F80018380003E0000C38000780000278000F00000178001E000000F8003C00000
0F800780000007800780000003800F00000003801F00000001801E00000001803E000000
01803C00000001803C00000000807C00000000807C0000000080780000000000F8000000
0000F80000000000F80000000000F80000000000F80000000000F80000000000F8000000
0000F80000000000F80000000000F80000000000F800000FFFFC7800000FFFFC7C000000
1FC07C0000000F803C0000000F803C0000000F803E0000000F801E0000000F801F000000
0F800F0000000F80078000000F8007C000000F8003C000000F8001E000000F8000F00000
1780007C00001780003E00006380000F8000C3800007F00781800000FFFE008000001FF0
00002E337CB134>71 D<FFFF80007FF0FFFF80007FF007F000001F8003E000001E0003E0
0000180003E00000100003E00000200003E00000400003E00000800003E00001000003E0
0002000003E00004000003E00008000003E00010000003E00020000003E00040000003E0
0080000003E00100000003E00200000003E00600000003E00F00000003E01F80000003E0
2F80000003E047C0000003E087E0000003E103E0000003E201F0000003E401F8000003E8
00F8000003F000FC000003E0007C000003E0003E000003E0003F000003E0001F000003E0
000F800003E0000FC00003E00007C00003E00003E00003E00003F00003E00001F00003E0
0000F80003E00000FC0003E000007C0003E000007E0003E000003E0003E000003F0007F0
00007FC0FFFF8001FFF8FFFF8001FFF82D317CB034>75 D<FFFFC00000FFFFC0000007F0
00000003E000000003E000000003E000000003E000000003E000000003E000000003E000
000003E000000003E000000003E000000003E000000003E000000003E000000003E00000
0003E000000003E000000003E000000003E000000003E000000003E000000003E0000000
03E000000003E000000003E000000003E000000003E000000003E000000003E000000003
E000002003E000002003E000002003E000002003E000004003E000004003E000004003E0
00004003E00000C003E00000C003E00001C003E00001C003E00003C003E000078003E000
1F8007E0007F80FFFFFFFF80FFFFFFFF8023317DB029>I<FFE00007FFC0FFF00007FFC0
03F000007C0002F80000380002FC00001000027C00001000023E00001000023E00001000
021F00001000021F80001000020F800010000207C00010000207E00010000203E0001000
0203F00010000201F00010000200F80010000200FC00100002007C00100002007E001000
02003E00100002001F00100002001F80100002000F80100002000FC01000020007C01000
020003E01000020003F01000020001F01000020000F81000020000F810000200007C1000
0200007E10000200003E10000200001F10000200001F10000200000F90000200000FD000
02000007D00002000003F00002000003F00002000001F00002000001F00002000000F000
0200000070000700000070000F8000003000FFF800003000FFF8000010002A317CB032>
78 D<FFFFFFE000FFFFFFFC0007E0003F0003E0000F8003E00003C003E00001E003E000
01F003E00000F803E00000F803E000007803E000007C03E000007C03E000007C03E00000
7C03E000007C03E000007C03E000007803E00000F803E00000F003E00001F003E00001E0
03E00003C003E0000F8003E0003E0003FFFFF80003E000000003E000000003E000000003
E000000003E000000003E000000003E000000003E000000003E000000003E000000003E0
00000003E000000003E000000003E000000003E000000003E000000003E000000003E000
000003E000000003E000000003E000000007F0000000FFFF800000FFFF80000026317DB0
2D>80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E0
700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8000000
7C0000007E0000003F0000003FC000001FF800000FFF800007FFF00003FFFC0000FFFF00
000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F8
8000007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0
F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83
D<FFFF8007FFC0FFFF8007FFC007F000007C0003E00000380003E00000100003E0000010
0003E00000100003E00000100003E00000100003E00000100003E00000100003E0000010
0003E00000100003E00000100003E00000100003E00000100003E00000100003E0000010
0003E00000100003E00000100003E00000100003E00000100003E00000100003E0000010
0003E00000100003E00000100003E00000100003E00000100003E00000100003E0000010
0003E00000100003E00000100003E00000100003E00000100003E00000100003E0000010
0003E00000100001E00000200001F00000200001F00000200000F00000400000F0000040
00007800008000003C00018000003C00010000001E000600000007800C00000003E07800
000000FFF0000000003F8000002A327CB032>85 D<FFFE003FFF8001FFC0FFFE003FFF80
01FFC00FE00003F800007E0007C00001F800001C0007C00001F80000180003E00000F800
00100003E00000F80000100003E00000FC0000100001F00000FC0000200001F00000FC00
00200001F00000FC0000200001F800013E0000200000F800013E0000400000F800013E00
00400000F800021F00004000007C00021F00008000007C00021F00008000007C00040F80
008000003E00040F80010000003E00040F80010000003E00040FC0010000001F000807C0
020000001F000807C0020000001F000807C0020000001F801003E0060000000F801003E0
040000000F801003E0040000000FC02001F00400000007C02001F00800000007C02001F0
0800000007C04000F80800000003E04000F81000000003E04000F81000000003E0C000FC
1000000001F080007C2000000001F080007C2000000001F080007E2000000001F900003E
6000000000F900003E4000000000F900003E4000000000FE00001F40000000007E00001F
80000000007E00001F80000000007C00000F80000000003C00000F00000000003C00000F
000000000038000007000000000018000006000000000018000006000000000018000006
00000042327FB044>87 D<000000F00000001FF00000001FF000000001F000000000F000
000000F000000000F000000000F000000000F000000000F000000000F000000000F00000
0000F000000000F000000000F000000000F000000000F000000000F000000000F000000F
C0F000007030F00001C00CF000038002F000070001F0000E0001F0001E0000F0003C0000
F0003C0000F0007C0000F000780000F000780000F000F80000F000F80000F000F80000F0
00F80000F000F80000F000F80000F000F80000F000780000F000780000F0007C0000F000
3C0000F0003C0000F0001E0000F0000E0001F000070002F000038004F00001C008F80000
F030FF80001FC0FF8021327EB125>100 D<003F800000E0E0000380380007003C000E00
1E001E001E001C000F003C000F007C000F0078000F8078000780F8000780F8000780FFFF
FF80F8000000F8000000F8000000F8000000F8000000F8000000780000007C0000003C00
00003C0000801E0000800E0001000F0002000700020001C00C0000F03000001FC000191F
7E9E1D>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C001E003C
003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C000F0078
00070070000780E00009C1C000087F000018000000180000001800000018000000180000
001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000F0700000
70E0000038E0000038E0000038E0000038E00000387000007070000070380000E01C0001
C00700070001C01C00003FE0001E2F7E9F21>103 D<0780000000FF80000000FF800000
000F80000000078000000007800000000780000000078000000007800000000780000000
078000000007800000000780000000078000000007800000000780000000078000000007
8000000007800000000780FE00000783078000078C03C000079001E00007A001E00007A0
00F00007C000F00007C000F000078000F000078000F000078000F000078000F000078000
F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0
00078000F000078000F000078000F000078000F000078000F000078000F000078000F000
078000F0000FC001F800FFFC1FFF80FFFC1FFF8021327EB125>I<0F001F801F801F801F
800F000000000000000000000000000000000000000000000007807F807F800F80078007
800780078007800780078007800780078007800780078007800780078007800780078007
8007800780078007800FC0FFF8FFF80D307EAF12>I<07800000FF800000FF8000000F80
000007800000078000000780000007800000078000000780000007800000078000000780
000007800000078000000780000007800000078000000780000007800FFE07800FFE0780
03F007800380078003000780060007800800078010000780200007804000078080000781
800007838000078FC0000793C00007A1E00007C1F0000780F0000780780007807C000780
3C0007803E0007801F0007800F0007800F80078007C0078003C0078003E00FC007F8FFFC
0FFFFFFC0FFF20327EB123>107 D<0780FE0000FF83078000FF8C03C0000F9001E00007
A001E00007A000F00007C000F00007C000F000078000F000078000F000078000F0000780
00F000078000F000078000F000078000F000078000F000078000F000078000F000078000
F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0
00078000F000078000F0000FC001F800FFFC1FFF80FFFC1FFF80211F7E9E25>110
D<001FC00000F0780001C01C00070007000F0007801E0003C01C0001C03C0001E03C0001
E0780000F0780000F0780000F0F80000F8F80000F8F80000F8F80000F8F80000F8F80000
F8F80000F8F80000F8780000F07C0001F03C0001E03C0001E01E0003C01E0003C00F0007
8007800F0001C01C0000F07800001FC0001D1F7E9E21>I<0783E0FF8418FF887C0F907C
07A07C07A03807C00007C00007C000078000078000078000078000078000078000078000
078000078000078000078000078000078000078000078000078000078000078000078000
0FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030
E00010E00010E00010F00010F800007E00003FF0001FFF000FFFC003FFE0003FF00001F8
0000F880003C80003C80001CC0001CC0001CE0001CE00018F00038F00030CC0060C301C0
80FE00161F7E9E1A>I<00400000400000400000400000400000C00000C00000C00001C0
0001C00003C00007C0000FC0001FFFE0FFFFE003C00003C00003C00003C00003C00003C0
0003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C01003C0
1003C01003C01003C01003C01003C01003C01001C02001E02000E0400078C0001F00142C
7FAB19>I<078000F000FF801FF000FF801FF0000F8001F000078000F000078000F00007
8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780
00F000078000F000078000F000078000F000078000F000078000F000078000F000078000
F000078000F000078000F000078001F000078001F000078001F000038002F00003C004F0
0001C008F800007030FF80001FC0FF80211F7E9E25>I<FFF800FF80FFF800FF800FC000
7C000780003800078000300003C000200003C000200001E000400001E000400001F00040
0000F000800000F000800000780100000078010000007C010000003C020000003C020000
001E040000001E040000001F040000000F080000000F0800000007900000000790000000
07D000000003E000000003E000000001C000000001C000000001C000000000800000211F
7F9E23>I E end
%%EndProlog
%%BeginSetup
%%Feature: *Resolution 300dpi
TeXDict begin

%%EndSetup
%%Page: 1 1
1 0 bop 655 381 a FB(LAP)-6 b(A)n(CK)22 b(W)-6 b(orking)22
b(Note)g(94)532 473 y(A)f(User's)h(Guide)f(to)h(the)g(BLA)n(CS)f(v1.1)
1568 447 y FA(\003)607 599 y Fz(Jac)o(k)15 b(J.)h(Dongarra,)1003
581 y Fy(y)1125 599 y Fz(R.)g(Clin)o(t)f(Whaley)1493
581 y Fy(z)919 701 y Fz(Ma)o(y)h(5,)g(1997)959 919 y
Fx(Abstract)326 994 y Fw(The)e(BLA)o(CS)f(\(Basic)h(Linear)f(Algebra)g
(Comm)o(uni)o(cation)d(Subprograms\))j(pro)r(ject)h(is)f(an)g(on-)264
1044 y(going)d(in)o(v)o(estigation)h(whose)i(purp)q(ose)g(is)f(to)g
(create)h(a)f(linear)g(algebra)f(orien)o(ted)i(message)e(passing)264
1094 y(in)o(terface)f(that)g(is)g(implemen)o(ted)d(e\016cien)o(tly)j
(and)g(uniformly)d(across)k(a)f(large)f(range)h(of)g(distributed)264
1144 y(memory)h(platforms.)326 1194 y(The)16 b(length)g(of)g(time)e
(required)j(to)f(implemen)o(t)d(e\016cien)o(t)k(distributed)f(memory)e
(algorithms)264 1243 y(mak)o(es)f(it)h(impractical)e(to)i(rewrite)i
(programs)d(for)h(ev)o(ery)h(new)g(parallel)e(mac)o(hine.)18
b(The)d(BLA)o(CS)264 1293 y(exist)k(in)f(order)i(to)f(mak)o(e)e(linear)
i(algebra)f(applications)g(b)q(oth)h(easier)h(to)e(program)g(and)g
(more)264 1343 y(p)q(ortable.)326 1393 y(It)e(is)h(for)f(this)g(reason)
h(that)g(the)g(BLA)o(CS)g(are)g(used)g(as)g(the)g(comm)o(uni)o(cation)d
(la)o(y)o(er)i(for)g(the)264 1443 y(ScaLAP)m(A)o(CK)c(pro)r(ject,)h
(whic)o(h)f(in)o(v)o(olv)o(es)f(implemen)o(ting)e(the)j(LAP)m(A)o(CK)h
(library)e(on)h(distributed)264 1493 y(memory)f(MIMD)j(mac)o(hines.)326
1542 y(This)g(rep)q(ort)h(describ)q(es)h(the)e(library)f(whic)o(h)h
(has)g(arisen)g(from)e(this)i(pro)r(ject.)p 150 2430
720 2 v 201 2457 a Fv(\003)219 2473 y Fu(This)k(w)o(ork)f(w)o(as)f
(supp)q(orted)j(in)e(part)h(b)o(y)f(D)o(ARP)m(A)g(and)g(AR)o(O)g(under)
h(con)o(tract)f(n)o(um)o(b)q(er)h(D)o(AAL03-91-C-0047,)150
2519 y(and)12 b(in)h(part)e(b)o(y)h(the)g(National)h(Science)g(F)m
(oundation)h(Science)f(and)f(T)m(ec)o(hnology)h(Cen)o(ter)e(Co)q(op)q
(erativ)o(e)j(Agreemen)o(t)d(No.)150 2564 y(CCR-8809615.)203
2595 y Fv(y)219 2611 y Fu(Dept.)23 b(of)14 b(Computer)h(Sciences,)i
(Univ.)23 b(of)14 b(TN,)g(Kno)o(xville,)j(TN)d(37996,)i(and)f
(Mathematical)i(Sciences)g(Section,)150 2657 y(ORNL,)12
b(Oak)i(Ridge,)g(TN)e(37831,)h Ft(dongarra@c)o(s.u)o(tk)o(.ed)o(u)203
2688 y Fv(z)219 2704 y Fu(Dept.)18 b(of)12 b(Computer)i(Sciences,)g
(Univ.)k(of)12 b(TN,)g(Kno)o(xville,)k(TN)c(37996,)i
Ft(rwhaley@)o(cs.)o(utk)o(.e)o(du)1044 2828 y Fs(i)p
eop
%%Page: 2 2
2 1 bop 150 199 a Fr(Con)n(ten)n(ts)150 301 y Fq(1)42
b(In)o(tro)q(duction)1419 b(1)150 404 y(2)42 b(Av)m(ailabilit)o(y)20
b(of)d(the)h(BLA)o(CS)f(Soft)o(w)o(are)886 b(2)150 506
y(3)42 b(What's)18 b(new)f(for)g(this)h(release)1079
b(3)150 608 y(4)42 b(BLA)o(CS)17 b(Concepts)h(and)g(F)l(eatures)992
b(3)218 664 y Fs(4.1)46 b(Arra)o(y-based)15 b(Comm)o(unication)45
b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)218
721 y(4.2)46 b(Pro)q(cess)15 b(Grid)g(and)h(Scop)q(ed)g(Op)q(erations)
26 b Fp(:)c(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(4)218 778 y(4.3)46 b(Con)o(texts)14
b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)
f(:)g(:)h(:)f(:)91 b Fs(5)218 834 y(4.4)46 b(ID-less)16
b(Comm)o(unication)46 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)91 b Fs(6)218 891 y(4.5)46 b(Blo)q(c)o(king)16 b(Lev)o(els)25
b Fp(:)e(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91
b Fs(7)150 993 y Fq(5)42 b(Naming)18 b(Con)o(v)o(en)o(tions)1228
b(8)150 1095 y(6)42 b(P)o(oin)o(t)17 b(T)l(o)h(P)o(oin)o(t)f(Comm)o
(unication)990 b(8)218 1152 y Fs(6.1)46 b(Seman)o(tics)29
b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)
g(:)h(:)f(:)91 b Fs(8)218 1208 y(6.2)46 b(Syn)o(tax)19
b Fp(:)j(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)
h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)69 b Fs(11)323 1265 y(6.2.1)50 b(P)o(oin)o(t)15
b(to)g(P)o(oin)o(t)f(Sends)24 b Fp(:)e(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(11)323 1322 y(6.2.2)50 b(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f
(Receiv)o(es)41 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(12)218 1378 y(6.3)46 b(Example)19 b Fp(:)j(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)
h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(13)150 1480 y Fq(7)42 b(Broadcasts)1431 b(13)218
1537 y Fs(7.1)46 b(Seman)o(tics)29 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b
Fs(13)218 1594 y(7.2)46 b(Syn)o(tax)19 b Fp(:)j(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(13)323 1650 y(7.2.1)50 b(Broadcast/send)19 b Fp(:)j(:)g(:)h(:)f(:)
g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(14)323 1707 y(7.2.2)50
b(Broadcast/receiv)o(e)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(15)218 1763 y(7.3)46 b(Example)19 b Fp(:)j(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(16)218 1820 y(7.4)46 b(T)l(op)q(ologies)18 b Fp(:)23
b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(16)150 1922 y Fq(8)42 b(Com)o(bines)1456
b(17)218 1979 y Fs(8.1)46 b(Seman)o(tics)29 b Fp(:)23
b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(17)218 2035 y(8.2)46 b(Syn)o(tax)19 b Fp(:)j(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(18)218 2092 y(8.3)46 b(Example)19 b Fp(:)j(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)
h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(19)218 2149 y(8.4)46 b(T)l(op)q(ologies)18 b Fp(:)23
b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(20)150 2251 y Fq(9)42 b(Supp)q(ort)18 b(Routines)1275
b(20)218 2307 y Fs(9.1)46 b(Initialization)h Fp(:)22
b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(20)323 2364 y(9.1.1)50 b(BLA)o(CS)p 622 2364 14
2 v 17 w(PINF)o(O)17 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)
h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)69 b Fs(20)323 2421 y(9.1.2)50 b(BLA)o(CS)p 622
2421 V 17 w(SETUP)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)69 b Fs(21)323 2477 y(9.1.3)50 b(BLA)o(CS)p 622 2477
V 17 w(GRIDINIT)13 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(21)323 2534 y(9.1.4)50 b(BLA)o(CS)p 622 2534 V 17
w(GRIDMAP)41 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(22)218 2590 y(9.2)46 b(Destruction)31 b Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(24)323 2647 y(9.2.1)50 b(BLA)o(CS)p 622 2647
V 17 w(FREEBUFF)17 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(24)323 2704 y(9.2.2)50 b(BLA)o(CS)p 622 2704 V 17
w(GRIDEXIT)34 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(24)1037 2828 y(ii)p eop
%%Page: 3 3
3 2 bop 323 199 a Fs(9.2.3)50 b(BLA)o(CS)p 622 199 14
2 v 17 w(ABOR)l(T)34 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)69 b Fs(25)323 256 y(9.2.4)50 b(BLA)o(CS)p 622 256
V 17 w(EXIT)12 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(25)218 313 y(9.3)46 b(Informational)15 b(and)h
(Miscellaneous)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(25)323 370 y(9.3.1)50 b(BLA)o(CS)p 622 370 V 17
w(GRIDINF)o(O)34 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(25)323 427 y(9.3.2)50 b(BLA)o(CS)p 622 427 V 17
w(PNUM)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(26)323 484 y(9.3.3)50 b(BLA)o(CS)p 622 484 V 17
w(PCOORD)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(26)323 541 y(9.3.4)50 b(BLA)o(CS)p 622 541 V 17
w(BARRIER)20 b Fp(:)j(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(27)218 597 y(9.4)46 b(General)15 b(purp)q(ose)45
b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(27)323 654 y(9.4.1)50 b(BLA)o(CS)p 622 654 V 17
w(GET)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(27)323 711 y(9.4.2)50 b(BLA)o(CS)p 622 711 V 17
w(SET)37 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(28)218 768 y(9.5)46 b(Uno\016cial)16 b(routines)42
b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(30)323 825 y(9.5.1)50 b(SETPVMTIDS)25 b Fp(:)d(:)g(:)h(:)f(:)g(:)h
(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(31)323 882 y(9.5.2)50
b(DCPUTIME)39 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(31)323 939 y(9.5.3)50 b(D)o(W)-5 b(ALL)l(TIME)46
b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(31)323 996 y(9.5.4)50 b(KSENDID)13 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(32)323 1053
y(9.5.5)50 b(KRECVID)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)
f(:)g(:)h(:)f(:)69 b Fs(32)323 1109 y(9.5.6)50 b(KBSID)45
b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(32)323 1166 y(9.5.7)50 b(KBRID)37 b Fp(:)23 b(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(32)150
1269 y Fq(REFERENCES)1393 b(34)150 1372 y(A)28 b(C)17
b(In)o(terface)h(to)f(the)h(BLA)o(CS)1077 b(36)218 1429
y Fs(A.1)35 b(Supp)q(ort)15 b(Routines)20 b Fp(:)i(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(36)323 1485
y(A.1.1)39 b(Initialization)k Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)
f(:)g(:)h(:)f(:)69 b Fs(36)323 1542 y(A.1.2)39 b(Destruction)27
b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(36)323 1599 y(A.1.3)39 b(Informational)16 b(and)f(Miscellaneous)39
b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f
(:)g(:)h(:)f(:)69 b Fs(36)323 1656 y(A.1.4)39 b(Uno\016cial)e
Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(37)218 1713 y(A.2)35 b(P)o(oin)o(t)15 b(to)f(P)o(oin)o(t)g
Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(37)218 1770 y(A.3)35 b(Broadcasts)45 b Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(37)218 1827 y(A.4)35 b(Com)o(bines)f Fp(:)23
b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(37)150 1929 y Fq(B)31 b(Degrees)17 b(of)g(Blo)q(c)o(king)
1226 b(38)218 1986 y Fs(B.1)37 b(Non-blo)q(c)o(king)17
b(comm)o(unication)37 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(38)218 2043 y(B.2)37 b(Lo)q(cally-blo)q(c)o(king)47
b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(39)218 2100 y(B.3)37 b(Globally-blo)q(c)o(king)23
b Fp(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)
h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(40)150 2203 y Fq(C)30 b(BLA)o(CS)17 b(Error)g(Handling)1139
b(41)218 2260 y Fs(C.1)36 b(BLA)o(CS)15 b(W)l(arning)h(and)f(Error)f
(Messages)h Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(42)323 2317
y(C.1.1)40 b(Examples)33 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(42)218 2373 y(C.2)36 b(System)15
b(Error)f(Messages)f Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(43)323 2430 y(C.2.1)40 b(Examples)33 b Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(43)150 2533 y Fq(D)28 b(Rep)q(eatabili)q(t)o(y)19
b(and)f(coherence)1030 b(44)218 2590 y Fs(D.1)34 b(Rep)q(eatabilit)o(y)
f Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)
f(:)69 b Fs(44)218 2647 y(D.2)34 b(Coherence)23 b Fp(:)g(:)f(:)g(:)h(:)
f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(45)323 2704 y(D.2.1)38 b(Example)16 b(of)f(Incoherence)33
b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(45)1031 2828
y(iii)p eop
%%Page: 4 4
4 3 bop 323 199 a Fs(D.2.2)38 b(Example)16 b(of)f(Homogeneous)g
(Coherence)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)69 b Fs(46)323 256 y(D.2.3)38 b(Example)16
b(of)f(Heterogeneous)g(Coherence)42 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(46)218
312 y(D.3)34 b(Summing)16 b(it)f(up)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)
h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(46)150
414 y Fq(E)34 b(Broadcast)18 b(T)l(op)q(ologies)1195
b(48)218 471 y Fs(E.1)38 b(Broadcast)14 b(Ring)i(T)l(op)q(ologies)d
Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(49)218
527 y(E.2)38 b(Broadcast)14 b(T)l(ree)i(T)l(op)q(ologies)j
Fp(:)k(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(52)150
629 y Fq(F)35 b(Com)o(bine)17 b(T)l(op)q(ologies)1221
b(56)218 685 y Fs(F.1)39 b(General)15 b(T)l(ree)h(Gather)27
b Fp(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(56)218 742 y(F.2)39 b(Bidirectional)17 b(Exc)o(hange)f
Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(56)150 844 y Fq(G)27 b(Multiring)19 b(Com)o(bine)1240
b(58)150 946 y(H)27 b(Example)18 b(Program)1263 b(60)150
1089 y Fr(List)22 b(of)h(T)-6 b(ables)218 1190 y Fs(1)82
b(Presen)o(tly)15 b(supp)q(orted)h(message)f(passing)g(la)o(y)o(ers)38
b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h
(:)f(:)91 b Fs(2)218 1247 y(2)82 b(Scop)q(es)16 b(pro)o(vided)g(b)o(y)f
(a)g(2D)f(pro)q(cess)i(grid)d Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91
b Fs(5)218 1303 y(3)82 b(V)l(alues)16 b(and)f(meanings)h(of)f(the)g
(comm)o(unication)h(routines')f(name)g(p)q(ositions)32
b Fp(:)22 b(:)h(:)f(:)91 b Fs(9)218 1360 y(4)82 b(V)l(alues)16
b(and)f(meanings)h(of)f(com)o(bine)h(routines')f(name)g(p)q(ositions)37
b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)218
1416 y(5)82 b(Pre\014x)15 b(to)g(t)o(yp)q(e)g(declaration)h(mapping)25
b Fp(:)e(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)
f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(11)218 1473 y(6)82 b(Pre\014x)15
b(to)g(C)g(t)o(yp)q(e)g(declaration)h(mapping)d Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f
(:)g(:)h(:)f(:)69 b Fs(36)218 1529 y(7)82 b(Broadcast)14
b(top)q(ology)h(highligh)o(ts)23 b Fp(:)f(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(49)150 1672 y Fr(List)22 b(of)h(Figures)218 1774
y Fs(1)82 b(8)15 b(pro)q(cesses)g(mapp)q(ed)h(to)f(a)g(2)f(x)i(4)f(pro)
q(cess)g(grid.)34 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(4)218 1830 y(2)82
b(After)15 b(\014rst)f(step)i(of)e(LU)i(factorization)35
b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h
(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)218 1887 y(3)82
b(Increasing)16 b(ring)g(broadcast)42 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h
(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)
f(:)g(:)h(:)f(:)69 b Fs(50)218 1943 y(4)82 b(Decreasing)15
b(ring)h(broadcast)29 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)
69 b Fs(50)218 2000 y(5)82 b(Split)16 b(ring)g(broadcast)44
b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(50)218 2056 y(6)82 b(Multiring)16 b(broadcast)f(with)g
Fp(N)874 2063 y Fo(r)905 2056 y Fs(=)e(3)34 b Fp(:)22
b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(51)218 2112 y(7)82 b(Hyp)q(ercub)q(e)17
b(broadcast,)d(nearest)h(no)q(de)h(\014rst.)j Fp(:)k(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(53)218 2169 y(8)82 b(General)15 b(tree)g(broadcast)g(with)g
Fp(N)931 2176 y Fo(b)961 2169 y Fs(=)e(1)49 b Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f
(:)g(:)h(:)f(:)69 b Fs(54)218 2225 y(9)82 b(General)15
b(tree)g(broadcast)g(with)g Fp(N)931 2232 y Fo(b)961
2225 y Fs(=)e(2)g Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g
(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(54)218
2282 y(10)59 b(General)15 b(tree)g(broadcast)g(with)g
Fp(N)931 2289 y Fo(b)961 2282 y Fs(=)e(3)g Fp(:)23 b(:)f(:)g(:)h(:)f(:)
g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69
b Fs(55)218 2338 y(11)59 b(General)15 b(tree)g(gather)g(with)g
Fp(N)865 2345 y Fo(b)895 2338 y Fs(=)e(1)44 b Fp(:)22
b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(57)218 2395 y(12)59 b(General)15
b(tree)g(gather)g(with)g Fp(N)865 2402 y Fo(b)895 2395
y Fs(=)e(4)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(57)218
2451 y(13)59 b(Bidirectional)17 b(exc)o(hange)27 b Fp(:)22
b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f
(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(58)1032
2828 y(iv)p eop
%%Page: 1 5
1 4 bop 150 199 a Fr(1)69 b(In)n(tro)r(duction)150 305
y Fs(The)17 b(BLA)o(CS)g(\(Basic)g(Linear)h(Algebra)f(Comm)o(unication)
g(Subprograms\))f([8)o(,)h(10)o(,)g(16)o(])f(is)i(a)e(pac)o(k)m(age)150
361 y(that)c(attempts)g(to)h(pro)o(vide)g(the)g(same)g(ease)g(of)g(use)
g(and)g(p)q(ortabilit)o(y)h(for)f(distributed)h(memory)f(linear)150
418 y(algebra)i(comm)o(unication)h(that)f(the)g(BLAS)h([5)o(,)f(6,)g
(15)o(])g(pro)o(vide)h(for)e(linear)j(algebra)e(computation.)221
476 y(The)i(concept)h(of)f(concen)o(trating)g(the)h(most)e(used)i
(computation)f(in)o(to)h(a)f(k)o(ernel)h(of)f(highly)i(opti-)150
533 y(mized)d(routines,)g(suc)o(h)g(as)f(the)g(BLAS,)h(has)f(pro)o(v)o
(en)g(itself)h(in)h(w)o(ork)d(on)h(LAP)l(A)o(CK)h([2,)f(1)o(].)20
b(LAP)l(A)o(CK)150 589 y(\(Linear)14 b(Algebra)f(P)l(A)o(CKage\))g(pro)
o(vides)g(linear)i(algebra)e(routines)h(for)e(sequen)o(tial)i(and)g
(shared)f(mem-)150 646 y(ory)i(mac)o(hines.)221 704 y(When)i(the)g
(ScaLAP)l(A)o(CK)g([9,)f(4])g(pro)s(ject)g(\(whic)o(h)h(in)o(v)o(olv)o
(es)h(p)q(orting)f(LAP)l(A)o(CK)g(to)f(distributed)150
761 y(memory)g(parallel)i(mac)o(hines\))f(w)o(as)e(b)q(egun,)j(it)f
(rapidly)g(b)q(ecame)g(eviden)o(t)h(that)e(a)g(similar)h(k)o(ernel)h
(for)150 817 y(comm)o(unication)e(w)o(ould)f(b)q(e)h(required.)21
b(Out)16 b(of)f(this)g(need)h(the)g(BLA)o(CS)f(arose.)221
876 y(With)h(these)h(t)o(w)o(o)e(k)o(ernels)i(in)h(place,)f(soft)o(w)o
(are)e(for)h(dense)h(linear)h(algebra)e(on)h(MIMD)e(platforms)150
932 y(can)c(consist)g(of)g(calls)h(to)e(the)h(BLAS)h(for)e(computation)
h(and)g(calls)i(to)d(the)h(BLA)o(CS)g(for)g(comm)o(unication.)150
989 y(Since)h(b)q(oth)f(pac)o(k)m(ages)g(will)h(ha)o(v)o(e)e(b)q(een)i
(optimized)g(for)e(that)g(particular)h(platform,)g(go)q(o)q(d)g(p)q
(erformance)150 1045 y(should)18 b(b)q(e)f(ac)o(hiev)o(ed)h(with)f
(relativ)o(ely)g(little)h(e\013ort.)23 b(Also,)17 b(since)h(b)q(oth)f
(pac)o(k)m(ages)g(will)h(b)q(e)f(a)o(v)m(ailable)150
1102 y(on)g(a)f(wide)h(v)m(ariet)o(y)g(of)g(mac)o(hines,)g(co)q(de)g
(mo)q(di\014cations)h(required)g(to)e(c)o(hange)g(platforms)h(should)g
(b)q(e)150 1158 y(minimal.)221 1217 y(There)12 b(are)h(v)m(arious)g
(pac)o(k)m(ages)f(designed)i(to)e(pro)o(vide)h(a)f(message)g(passing)h
(in)o(terface)g(that)f(remains)150 1273 y(unc)o(hanged)20
b(across)f(m)o(ultiple)j(platforms,)e(including)i(PICL)e([13)o(],)g
(PVM)f([12)o(])h(and)f(more)g(recen)o(tly)l(,)150 1330
y(MPI)e([11)o(].)24 b(These)18 b(pac)o(k)m(ages)e(are)h(general)g
(libraries,)i(ho)o(w)o(ev)o(er,)d(and)h(th)o(us)f(their)i(in)o
(terfaces)f(are)f(not)150 1386 y(as)f(easily)h(usable)g(for)f(linear)h
(algebra)f(applications)i(as)e(w)o(e)g(w)o(ould)g(lik)o(e.)221
1445 y(In)22 b(con)o(trast,)g(since)h(the)f(audience)h(of)f(the)f(BLA)o
(CS)i(is)f(kno)o(wn,)h(the)f(in)o(terface)g(and)g(metho)q(ds)150
1501 y(of)c(using)i(the)f(routines)g(can)h(b)q(e)f(sp)q(ecialized)j
(\(and)d(th)o(us)g(simpli\014ed\).)33 b(F)l(or)18 b(example,)i(the)f
(BLA)o(CS)150 1558 y(are)h(written)g(at)f(a)h(lev)o(el)h(where)f(the)g
(manipulation)i(of)d(the)h(matrices)g(in)o(v)o(olv)o(ed)h(in)g(linear)g
(algebra)150 1614 y(computations)15 b(is)h(b)q(oth)f(natural)g(and)h
(con)o(v)o(enien)o(t.)221 1673 y(The)f(goals)g(of)g(the)g(BLA)o(CS)h
(pro)s(ject)e(include:)218 1775 y Fn(\017)23 b Fm(Ease)14
b(of)i(pr)n(o)n(gr)n(amming)d Fs(Wherev)o(er)h(p)q(ossible,)i(the)e
(BLA)o(CS)g(will)i(simplify)f(message)f(passing)g(in)264
1832 y(order)h(to)f(reduce)i(programming)f(errors.)218
1934 y Fn(\017)23 b Fm(Ease)15 b(of)h(use)f Fs(The)g(in)o(terface)h(to)
e(the)h(BLA)o(CS)h(will)g(b)q(e)g(at)e(suc)o(h)h(a)g(lev)o(el)h(as)f
(to)f(b)q(e)i(easily)g(usable)264 1991 y(b)o(y)f(linear)h(algebra)f
(programmers.)218 2093 y Fn(\017)23 b Fm(Portability)c
Fs(The)h(BLA)o(CS)g(m)o(ust)e(supply)j(an)e(in)o(terface)h(whic)o(h)g
(can)g(b)q(e)g(supp)q(orted)g(across)e(a)264 2150 y(wide)12
b(range)f(of)g(parallel)h(computers,)g(including)i(parallel)f(mac)o
(hines)f(built)g(from)f(heterogeneous)264 2206 y(pro)q(cessors.)221
2308 y(The)h(\014rst)g(section)h(of)f(this)h(rep)q(ort)f(discusses)h
(do)o(wnloading)g(and)g(a)o(v)m(ailabilit)o(y)h(issues.)20
b(The)13 b(follo)o(w-)150 2365 y(ing)k(section)f(familiarizes)i(the)e
(reader)g(with)h(some)f(of)f(the)h(more)g(imp)q(ortan)o(t)g(concepts)g
(and)g(features)150 2421 y(of)c(the)h(BLA)o(CS.)g(W)l(e)g(then)g
(discuss)h(the)f(four)g(main)g(categories)f(of)h(BLA)o(CS)g(routines.)
20 b(The)13 b(\014rst)f(cate-)150 2478 y(gory)g(consists)g(of)g(p)q
(oin)o(t)h(to)f(p)q(oin)o(t)h(message)f(passing.)19 b(Next,)13
b(broadcasts,)e(whic)o(h)j(tak)o(e)d(data)h(from)g(one)150
2534 y(pro)q(cess)j(and)f(send)h(it)g(to)f(man)o(y)g(pro)q(cesses,)g
(are)h(examined.)20 b(Then,)15 b(com)o(bines)g(are)f(discussed.)21
b(Com-)150 2591 y(bines)d(tak)o(e)e(data)g(distributed)i(o)o(v)o(er)e
(pro)q(cesses,)h(and)g(com)o(bine)h(the)e(data)g(in)i(some)e(w)o(a)o(y)
g(to)g(pro)q(duce)150 2647 y(a)g(result)h(\(at)f(presen)o(t,)h(data)f
(can)h(b)q(e)g(com)o(bined)h(b)o(y)e(summation)h(or)f(absolute)h(v)m
(alue)h(maximization)150 2704 y(or)h(minimization\).)33
b(Finally)l(,)22 b(w)o(e)d(discuss)h(the)g(supp)q(ort)f(routines,)h
(whic)o(h)g(p)q(erform)f(man)o(y)g(div)o(erse)1039 2828
y(1)p eop
%%Page: 2 6
2 5 bop 150 199 a Fs(functions,)21 b(often)e(not)g(directly)h(related)g
(to)f(comm)o(unication)h(\(for)e(example,)j(returning)f(a)f(pro)q(cess)
150 256 y(ID\).)221 312 y(The)i(app)q(endices)i(discuss)f(the)f(C)f(in)
o(terface)i(to)e(the)h(BLA)o(CS,)g(error)f(handling,)k(the)d(issues)g
(of)150 369 y(rep)q(eatabilit)o(y)15 b(and)e(coherence,)i(broadcast)d
(and)i(com)o(bine)g(top)q(ologies,)g(as)f(w)o(ell)i(as)e(pro)o(viding)h
(further)150 425 y(description)j(of)d(the)i(blo)q(c)o(king)g(lev)o(els)
h(outlined)g(in)f(Section)g(4.5.)221 482 y(This)f(user's)g(guide)h(is)g
(supplemen)o(ted)h(b)o(y)e(the)g(BLA)o(CS)h(w)o(eb)f(page.)20
b(The)15 b(URL)i(is)150 538 y Fl(http://www.netlib.org/blac)o(s/Blacs)o
(.html)p Fs(.)37 b(This)23 b(on-line)h(do)q(cumen)o(t)f(giv)o(es)f
(detailed)i(ex-)150 594 y(amples,)18 b(as)e(w)o(ell)i(as)e(pro)o
(viding)i(reference,)g(do)o(wnloading)f(options,)h(installation)g
(instructions,)g(and)150 651 y(troublesho)q(oting.)i(If)14
b(problems)g(still)g(remain)g(after)f(reading)h(this)g(guide)g(and)g
(consulting)g(the)g(mosaic)150 707 y(page,)h(questions)g(should)i(b)q
(e)e(mailed)i(to)e Fl(blacs@cs.utk.edu)p Fs(.)150 849
y Fr(2)69 b(Av)l(ailabilit)n(y)20 b(of)j(the)g(BLA)n(CS)g(Soft)n(w)n
(are)150 950 y Fs(The)13 b(BLA)o(CS)g(source)f(co)q(de)h(and)g(do)q
(cumen)o(tation)g(is)g(a)o(v)m(ailable)h(through)f Fm(netlib)p
Fs(.)18 b(Netlib)13 b(is)h(a)e(soft)o(w)o(are)150 1007
y(distribution)j(service)g(set)e(up)h(on)g(the)f(In)o(ternet)h(that)f
(con)o(tains)h(a)f(wide)h(range)f(of)h(computer)f(soft)o(w)o(are.)150
1063 y(Soft)o(w)o(are)h(can)h(b)q(e)h(retriev)o(ed)g(from)e(netlib)j(b)
o(y)e(h)o(ttp,)f(ftp,)h(or)g(email.)221 1120 y(A)o(t)i(presen)o(t,)i
(four)g(di\013eren)o(t)f(BLA)o(CS)h(implemen)o(tations)h(are)e(a)o(v)m
(ailable)i(from)e(netlib.)31 b(Eac)o(h)18 b(of)150 1176
y(these)h(four)f(BLA)o(CS)h(implemen)o(tations)h(is)f(based)g(on)g(a)f
(di\013eren)o(t)h(message)f(passing)i(la)o(y)o(er.)30
b(These)150 1233 y(message)21 b(passing)g(la)o(y)o(ers)g(and)h(the)f
(mac)o(hines)h(they)f(are)g(normally)h(supp)q(orted)f(on)g(are)g(sho)o
(wn)g(in)150 1289 y(T)l(able)16 b(1.)221 1346 y(In)i(addition)h(to)e
(the)h(BLA)o(CS)g(v)o(ersions)g(a)o(v)m(ailable)h(on)f(netlib)h([7)o
(],)f(sev)o(eral)g(v)o(endors)f(\(e.g.)27 b(Cra)o(y)l(,)150
1402 y(IBM)15 b(and)h(Meik)o(o\))f(are)f(presen)o(tly)i(pro)q(ducing)h
(optimized)f(v)o(ersions)g(for)e(their)i(mac)o(hines.)p
150 1462 1818 2 v 150 1472 V 149 1528 2 57 v 158 1528
V 183 1511 a(MESSA)o(GE)p 430 1528 V 1958 1528 V 1967
1528 V 149 1584 V 158 1584 V 183 1568 a(P)l(ASSING)p
430 1584 V 76 w(MA)o(CHINES)p 1958 1584 V 1967 1584 V
149 1641 V 158 1641 V 183 1624 a(LA)l(YER)p 430 1641
V 1958 1641 V 1967 1641 V 150 1643 1818 2 v 150 1653
V 149 1709 2 57 v 158 1709 V 183 1692 a(MPI)p 430 1709
V 184 w(Most)e(systems.)p 1958 1709 V 1967 1709 V 150
1711 1818 2 v 150 1721 V 149 1777 2 57 v 158 1777 V 183
1760 a(MPL)p 430 1777 V 172 w(IBM's)h(SP)g(series)h(\(SP1)e(and)i
(SP2\))p 1958 1777 V 1967 1777 V 150 1779 1818 2 v 149
1835 2 57 v 158 1835 V 183 1818 a(NX)p 430 1835 V 205
w(In)o(tel's)f(sup)q(ercomputer)h(series)g(\(iPSC2,)f(iPSC/860,)f(DEL)l
(T)l(A)i(and)f(P)l(ARA)o(GON\).)p 1958 1835 V 1967 1835
V 150 1837 1818 2 v 149 1893 2 57 v 158 1893 V 183 1876
a(PVM)p 430 1893 V 166 w(Most)f(systems.)p 1958 1893
V 1967 1893 V 150 1895 1818 2 v 150 1905 V 529 1982 a(T)l(able)i(1:)k
(Presen)o(tly)15 b(supp)q(orted)h(message)f(passing)g(la)o(y)o(ers)221
2083 y(NOTE:)h(In)h(the)g(past)f(w)o(e)h(ha)o(v)o(e)f(also)h(supp)q
(orted)g(a)f(CMMD)g(v)o(ersion)h(of)f(the)h(BLA)o(CS,)g(designed)150
2139 y(for)c(Thinking)i(Mac)o(hine's)f(CM-5.)19 b(It)14
b(app)q(ears)f(that)g(there)h(are)g(no)f(longer)h(users)g(for)f(this)h
(pac)o(k)m(age.)20 b(If)150 2195 y(y)o(ou)12 b(need)h(the)f(CMMDBLA)o
(CS,)g(send)g(mail)i(to)d Fl(blacs@cs.utk.edu)p Fs(,)f(and)j(w)o(e)f
(will)i(examine)f(making)150 2252 y(this)j(v)o(ersion)f(a)o(v)m
(ailable)i(again.)221 2308 y(The)10 b(BLA)o(CS)h(homepage)f(can)g(b)q
(e)h(accessed)f(at)g(URL)h Fl(http://www.netlib.org/blac)o(s/Blacs.)o
(html)p Fs(.)150 2365 y(The)19 b(BLA)o(CS)h(homepage)e(con)o(tains)i
(reference)f(to)f(the)i(routines,)f(examples,)i(installation)f
(instruc-)150 2421 y(tions,)15 b(troublesho)q(oting,)g(and)h(do)o
(wnloading)g(options.)221 2478 y(The)21 b(BLA)o(CS)h(\014les)g(ma)o(y)e
(b)q(e)i(obtained)g(via)f(anon)o(ymous)g(ftp)g(to)f Fl(netlib.org)p
Fs(.)37 b(Lo)q(ok)21 b(in)h(the)150 2534 y(directory)15
b Fl(blacs)p Fs(.)20 b(The)15 b(\014le)h Fl(index)f Fs(describ)q(es)i
(the)e(\014les)h(a)o(v)m(ailable)h(in)f(this)g(directory)l(.)221
2591 y(Finally)l(,)e(the)f(BLA)o(CS)g(ma)o(y)e(b)q(e)j(obtained)f(b)o
(y)f(email.)20 b(T)l(o)13 b(receiv)o(e)g(do)o(wnloading)g(instructions)
h(and)150 2647 y(a)19 b(list)h(of)e(a)o(v)m(ailable)j(\014les,)f(send)g
(email)g(to)f Fl(netlib@netlib.org)d Fs(with)k(the)f(message)f
Fl(send)24 b(index)150 2704 y(from)f(blacs)p Fs(.)1039
2828 y(2)p eop
%%Page: 3 7
3 6 bop 150 199 a Fr(3)69 b(What's)24 b(new)e(for)h(this)g(release)150
328 y Fs(This)15 b(release)g(is)g(really)g(just)f(a)g(minor)h(up)q
(date)g(from)f(the)g(1.0)g(release.)20 b(The)14 b(most)g(imp)q(ortan)o
(t)g(c)o(hange)150 385 y(is)i(that)e(there)i(are)f(no)o(w)g(\015ags)g
(whic)o(h)h(con)o(trol)f(whether)g(the)h(com)o(bine)g(op)q(erations)f
(are)g Fm(c)n(oher)n(ent)g Fs(and)150 441 y Fm(r)n(ep)n(e)n(atable)p
Fs(.)k(See)d(section)g(9.4)e(and)h(app)q(endix)i(D)e(for)g(details.)221
512 y(Also)f(w)o(orth)f(noting)i(is)f(that)g(the)g(setting)g(of)g(the)g
(n)o(um)o(b)q(er)h(of)f(branc)o(hes)g(\(rings\))g(for)g(tree)g(\(m)o
(ultir-)150 568 y(ing\))h(broadcast)e(has)i(b)q(een)g(sep)q(erated)g
(from)f(the)g(n)o(um)o(b)q(er)h(of)f(branc)o(hes)g(\(rings\))g(for)g
(tree)g(\(m)o(ultiring\))150 624 y(com)o(bine.)21 b(See)16
b(section)f(9.4)g(for)f(further)h(details.)150 848 y
Fr(4)69 b(BLA)n(CS)23 b(Concepts)f(and)i(F)-6 b(eatures)150
977 y Fs(In)13 b(general,)g(this)g(pap)q(er)g(refers)f(to)g(the)g
(basic)h(unit)g(of)f(execution)i(as)e(a)g Fm(pr)n(o)n(c)n(ess)p
Fs(.)17 b(A)c(pro)q(cess)f(is)h(a)f(thread)150 1033 y(of)17
b(execution)h(whic)o(h)g(minimally)h(includes)g(a)e(stac)o(k,)f
(registers,)h(and)h(memory)l(.)25 b(Multiple)19 b(pro)q(cesses)150
1090 y(ma)o(y)14 b(share)i(a)e Fm(pr)n(o)n(c)n(essor)p
Fs(.)19 b(The)d(term)e(pro)q(cessor)h(refers)g(to)g(the)g(actual)h
(hardw)o(are.)221 1160 y(In)c(the)f(BLA)o(CS,)h(eac)o(h)f(pro)q(cess)h
(is)g(treated)f(as)g(if)h(it)g(w)o(ere)f(a)g(pro)q(cessor:)18
b(the)11 b(pro)q(cess)h(m)o(ust)f(exist)h(for)150 1217
y(the)e(lifetime)i(of)e(the)g(BLA)o(CS)h(run,)g(and)f(its)h(execution)g
(should)g(only)g(e\013ect)f(other)g(pro)q(cesses')g(execution)150
1273 y(through)18 b(the)g(use)g(of)f(message)h(passing)g(calls.)29
b(With)18 b(this)h(in)f(mind,)i(w)o(e)d(use)i(the)f(term)f(pro)q(cess)h
(in)150 1330 y(all)g(sections)f(of)g(this)g(pap)q(er)g(except)h(those)e
(dealing)j(with)e(timings.)26 b(When)17 b(discussing)i(timings,)e(w)o
(e)150 1386 y(sp)q(ecify)k(pro)q(cessors)f(as)g(our)f(unit)i(of)f
(execution,)i(since)f(sp)q(eedup)h(will)f(b)q(e)g(largely)g(determined)
g(b)o(y)150 1443 y(actual)15 b(hardw)o(are)g(resources.)150
1645 y Fk(4.1)56 b(Arra)n(y-based)19 b(Comm)n(unication)150
1758 y Fs(Man)o(y)c(comm)o(unication)h(pac)o(k)m(ages)f(can)g(b)q(e)h
(classi\014ed)h(as)e(ha)o(ving)g(op)q(erations)h(based)f(on)g(one)h
(dimen-)150 1815 y(sional)21 b(arra)o(ys,)g(whic)o(h)g(are)g(the)f(mac)
o(hine)i(represen)o(tation)e(for)g(linear)i(algebra's)e
Fm(ve)n(ctor)h Fs(class.)36 b(In)150 1871 y(programming)14
b(linear)i(algebra)e(problems,)h(ho)o(w)o(ev)o(er,)e(it)i(is)g(more)f
(natural)h(to)f(express)g(all)i(op)q(erations)150 1927
y(in)g(terms)e(of)h(t)o(w)o(o)f(dimensional)j(matrices.)j(V)l(ectors)14
b(and)i(scalars)f(are,)f(of)h(course,)f(simply)j(sub)q(classes)150
1984 y(of)i(matrices.)31 b(On)20 b(computers,)f(a)g(linear)h(algebra)g
(matrix)e(is)i(represen)o(ted)f(b)o(y)g(a)g(t)o(w)o(o)f(dimensional)150
2040 y(arra)o(y)c(\(2D)g(arra)o(y\),)f(and)j(therefore)f(the)g(BLA)o
(CS)h(op)q(erate)f(on)g(2D)f(arra)o(ys.)221 2111 y(The)e(BLA)o(CS)g
(recognize)h(the)g(t)o(w)o(o)d(most)i(common)f(classes)i(of)f(matrices)
g(for)f(dense)i(linear)g(algebra.)150 2167 y(The)i(\014rst)f(of)g
(these)h(classes)g(consists)g(of)f Fm(gener)n(al)h(r)n(e)n(ctangular)f
Fs(matrices,)g(whic)o(h)h(in)h(mac)o(hine)f(storage)150
2224 y(are)20 b(2D)g(arra)o(ys)f(consisting)j(of)e Fl(M)g
Fs(ro)o(ws)g(and)h Fl(N)f Fs(columns,)j(with)d(a)h(leading)h
(dimension,)h Fl(LDA)p Fs(,)d(that)150 2280 y(determines)e(the)f
(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)f(successiv)o(e)i(elemen)o(ts)
f(of)g(a)f(matrix)h(ro)o(w)f(in)i(memory)e(\(the)150
2337 y(BLA)o(CS)g(assume)f(column-ma)s(jor)g(storage)e(of)i(arra)o
(ys\).)221 2407 y(The)f(second)g(class)g(of)g(matrices)g(recognized)h
(b)o(y)f(the)g(BLA)o(CS)g(are)f Fm(tr)n(ap)n(ezoidal)h
Fs(matrices.)20 b(T)l(rap)q(e-)150 2464 y(zoidal)g(arra)o(ys)f(are)g
(de\014ned)h(b)o(y)g Fl(M)p Fs(,)e Fl(N)p Fs(,)h(and)h
Fl(LDA)p Fs(,)e(as)h(ab)q(o)o(v)o(e,)h(but)g(they)f(also)g(ha)o(v)o(e)g
(the)h(parameters)150 2520 y Fl(UPLO)p Fs(,)d(whic)o(h)h(indicates)h
(whether)f(the)f(matrix)g(is)h(upp)q(er)h(or)e(lo)o(w)o(er)g(trap)q
(ezoidal,)i(and)f Fl(DIAG)p Fs(,)e(whic)o(h)150 2577
y(determines)g(if)g(the)f(diagonal)g(of)g(the)g(matrix)g(need)h(b)q(e)f
(comm)o(unicated.)21 b(T)l(riangular)15 b(matrices)h(are)e(a)150
2633 y(sub)q(class)i(of)f(trap)q(ezoidal,)h(so)f(these)g(matrices)g
(are)g(also)g(handled)i(b)o(y)e(the)g(BLA)o(CS.)221 2704
y(The)g(shap)q(e)h(of)f(the)g(trap)q(ezoid)h(to)e(b)q(e)i(sen)o(t)f(is)
h(determined)g(b)o(y)f Fl(M)p Fs(,)g Fl(N)p Fs(,)g(and)g
Fl(UPLO)p Fs(:)1039 2828 y(3)p eop
%%Page: 4 8
4 7 bop 712 193 a Fl(UPLO)p 832 210 2 57 v 115 w(M)13
b Fn(\024)g Fl(N)p 1122 210 V 181 w(M)g Fp(>)g Fl(N)p
687 212 726 2 v 687 222 V 712 390 a Fs(`)p Fl(U)p Fs(')p
832 464 2 242 v 948 274 a Fj(n)1045 334 y(m)903 409 y(n)c
Fi(\000)g Fj(m)h Fw(+)g(1)p 873 297 150 2 v 873 338 a
Fh(@)906 372 y(@)p 948 372 75 2 v 1022 372 2 75 v 1122
464 2 242 v 1343 274 a Fj(n)1403 372 y(m)1125 334 y(m)g
Fi(\000)f Fj(n)h Fw(+)f(1)p 1305 297 75 2 v 1304 372
2 75 v 1305 413 a Fh(@)1339 447 y(@)p 1379 447 2 150
v 687 465 726 2 v 712 634 a Fs(`)p Fl(L)p Fs(')p 832
707 2 242 v 835 653 a Fj(m)948 728 y(n)873 593 y(n)g
Fi(\000)g Fj(m)h Fw(+)g(1)p 872 690 2 75 v 873 691 150
2 v 873 616 75 2 v 948 657 a Fh(@)981 690 y(@)p 1122
707 2 242 v 1125 615 a Fj(m)1193 713 y(n)1260 653 y(m)g
Fi(\000)f Fj(n)h Fw(+)f(1)p 1162 690 2 75 v 1162 690
2 150 v 1163 691 75 2 v 1237 690 2 75 v 1163 582 a Fh(@)1196
615 y(@)221 793 y Fs(The)18 b(pac)o(king)g(of)f(arra)o(ys)g(\(if)g
(required\))i(so)e(that)g(they)h(ma)o(y)f(b)q(e)i(sen)o(t)e(e\016cien)o
(tly)i(is)g(handled)g(in-)150 849 y(ternally)c(b)o(y)g(the)f(BLA)o(CS,)
g(allo)o(wing)i(the)e(user)h(to)e(concen)o(trate)h(on)g(the)h(logical)h
(matrix,)d(rather)h(than)150 905 y(ho)o(w)h(the)g(data)g(is)g
(organized)h(in)g(the)f(mac)o(hine's)h(memory)l(.)150
1041 y Fk(4.2)56 b(Pro)r(cess)18 b(Grid)g(and)h(Scop)r(ed)f(Op)r
(erations)150 1132 y Fs(The)h Fp(N)284 1139 y Fo(p)322
1132 y Fs(pro)q(cesses)h(in)o(v)o(olv)o(ed)g(in)f(a)g(parallel)i(task)d
(or)g(group)h(are)g(often)f(presen)o(ted)i(to)e(the)h(user)g(as)150
1188 y(a)f(linear)i(arra)o(y)e(of)g(pro)q(cess)h(IDs,)h(lab)q(eled)h(0)
p Fp(;)8 b Fs(1)p Fp(;)g(:)g(:)f(:)t(;)h(N)1126 1195
y Fo(p)1157 1188 y Fn(\000)13 b Fs(1.)30 b(F)l(or)18
b(reasons)h(describ)q(ed)h(b)q(elo)o(w,)g(it)f(is)150
1245 y(often)14 b(more)f(con)o(v)o(enien)o(t)i(to)e(map)h(this)h(1-D)e
(arra)o(y)g(of)h Fp(N)1134 1252 y Fo(p)1167 1245 y Fs(pro)q(cesses)g
(in)o(to)g(a)g(logical)h(t)o(w)o(o)e(dimensional)150
1301 y(pro)q(cess)18 b(mesh,)f(or)g(grid.)27 b(This)18
b(grid)g(will)h(ha)o(v)o(e)e Fn(P)k Fs(pro)q(cess)c(ro)o(ws)g(and)g
Fn(Q)h Fs(pro)q(cess)f(columns,)i(where)150 1358 y Fn(P)13
b(\003)d(Q)20 b Fs(=)g Fp(N)377 1365 y Fo(g)416 1358
y Fn(\024)g Fp(N)508 1365 y Fo(p)528 1358 y Fs(.)32 b(A)20
b(pro)q(cess)g(can)f(no)o(w)g(b)q(e)h(referenced)h(b)o(y)e(its)h(co)q
(ordinates)f(within)i(the)f(grid)150 1414 y(\(indicated)e(b)o(y)e(the)h
(notation)f Fn(f)p Fp(p;)8 b(q)r Fn(g)p Fs(,)15 b(where)i(0)d
Fn(\024)h Fp(p)g(<)g Fn(P)t Fs(,)h(and)h(0)d Fn(\024)h
Fp(q)i(<)e Fn(Q)p Fs(\),)h(rather)g(than)g(a)h(single)150
1470 y(n)o(um)o(b)q(er.)j(An)c(example)g(of)f(suc)o(h)g(a)g(mapping)h
(is)f(sho)o(wn)g(in)h(Figure)g(1.)834 1602 y Fz(0)126
b(1)f(2)g(3)739 1706 y(0)739 1856 y(1)p 770 1614 153
2 v 770 1763 2 150 v 835 1703 a Fs(0)p 921 1763 V 770
1765 153 2 v 920 1614 V 920 1763 2 150 v 127 w(1)p 1071
1763 V 920 1765 153 2 v 1069 1614 V 1069 1763 2 150 v
126 w(2)p 1220 1763 V 1069 1765 153 2 v 1219 1614 V 1219
1763 2 150 v 127 w(3)p 1370 1763 V 1219 1765 153 2 v
770 1763 V 770 1913 2 150 v 835 1853 a(4)p 921 1913 V
770 1915 153 2 v 920 1763 V 920 1913 2 150 v 127 w(5)p
1071 1913 V 920 1915 153 2 v 1069 1763 V 1069 1913 2
150 v 126 w(6)p 1220 1913 V 1069 1915 153 2 v 1219 1763
V 1219 1913 2 150 v 127 w(7)p 1370 1913 V 1219 1915 153
2 v 518 2012 a(Figure)15 b(1:)20 b(8)15 b(pro)q(cesses)g(mapp)q(ed)h
(to)f(a)g(2)g(x)g(4)g(pro)q(cess)g(grid.)221 2132 y(An)i(op)q(eration)h
(whic)o(h)g(in)o(v)o(olv)o(es)g(more)f(than)g(just)g(a)g(sender)h(and)g
(a)f(receiv)o(er)h(is)g(called)h(a)e Fm(sc)n(op)n(e)n(d)150
2188 y Fs(op)q(eration.)29 b(All)19 b(pro)q(cesses)f(that)g
(participate)h(in)f(a)g(scop)q(ed)h(op)q(eration)f(are)g(said)g(to)g(b)
q(e)h(within)g(the)150 2245 y(op)q(eration's)c(scop)q(e.)221
2304 y(On)j(a)f(system)h(using)g(a)g(linear)h(arra)o(y)d(of)h(pro)q
(cesses,)i(the)f(only)g(natural)g(scop)q(e)g(is)g(all)h(pro)q(cesses.)
150 2360 y(Using)d(a)f(2D)f(grid,)h(w)o(e)g(ha)o(v)o(e)g(3)g(natural)g
(scop)q(es,)h(as)e(sho)o(wn)h(in)h(T)l(able)g(2.)221
2419 y(These)c(groupings)h(of)f(pro)q(cesses)h(are)f(of)g(particular)h
(in)o(terest)f(to)g(the)h(linear)g(algebra)g(programmer,)150
2475 y(since)h(distributed)h(data)e(decomp)q(ositions)h(of)f(a)g(2D)f
(arra)o(y)g(\(a)h(linear)h(algebra)f(matrix\))g(tend)g(to)g(follo)o(w)
150 2532 y(this)21 b(pro)q(cess)g(mapping.)38 b(F)l(or)20
b(instance,)j(all)f(of)e(a)h(distributed)h(matrix)f(ro)o(w)f(can)h(b)q
(e)g(found)g(on)g(a)150 2588 y(pro)q(cess)15 b(ro)o(w,)f(etc.)221
2647 y(Viewing)21 b(the)g(ro)o(ws/columns)f(of)g(the)h(pro)q(cess)f
(grid)h(as)f(essen)o(tially)i(autonomous)e(subsystems)150
2704 y(pro)o(vides)15 b(the)f(programmer)f(with)i(additional)h(lev)o
(els)f(of)f(parallelism.)22 b(Of)15 b(course,)f(ho)o(w)g(indep)q(enden)
o(t)1039 2828 y(4)p eop
%%Page: 5 9
5 8 bop 460 155 1180 2 v 459 212 2 57 v 468 212 V 493
195 a Fs(SCOPE)p 674 212 V 52 w(MEANING)p 1631 212 V
1640 212 V 460 213 1180 2 v 460 223 V 459 280 2 57 v
468 280 V 493 263 a(Ro)o(w)p 674 280 V 119 w(All)16 b(pro)q(cesses)g
(in)g(a)f(pro)q(cess)g(ro)o(w)f(participate.)p 1631 280
V 1640 280 V 460 281 1180 2 v 459 338 2 57 v 468 338
V 493 321 a(Column)p 674 338 V 51 w(All)i(pro)q(cesses)g(in)g(a)f(pro)q
(cess)g(column)h(participate.)p 1631 338 V 1640 338 V
460 340 1180 2 v 459 396 2 57 v 468 396 V 493 379 a(All)p
674 396 V 149 w(All)g(pro)q(cesses)g(in)g(the)f(pro)q(cess)h(grid)f
(participate.)p 1631 396 V 1640 396 V 460 398 1180 2
v 587 475 a(T)l(able)h(2:)k(Scop)q(es)c(pro)o(vided)g(b)o(y)f(a)g(2D)f
(pro)q(cess)i(grid)150 615 y(these)j(ro)o(ws)e(and)i(columns)g
(actually)h(are)e(will)i(dep)q(end)g(up)q(on)g(the)e(underlying)j
(hardw)o(are.)29 b(F)l(or)18 b(in-)150 671 y(stance,)h(if)g(the)f
(grid's)g(pro)q(cessors)g(are)g(connected)i(via)e(ethernet,)h(w)o(e)f
(can)h(see)g(that)e(the)i(only)g(gain)150 728 y(will)d(b)q(e)f(in)g
(ease)g(of)e(programming.)19 b(Sp)q(eed)d(is)f(unlik)o(ely)i(to)c
(increase,)i(since)h(if)f(one)f(pro)q(cessor)g(is)h(com-)150
784 y(m)o(unicating,)i(no)f(others)g(can.)24 b(If)16
b(this)h(is)g(the)f(case,)h(pro)q(cess)f(ro)o(ws)g(or)f(columns)j(will)
g(not)d(b)q(e)i(able)h(to)150 841 y(p)q(erform)12 b(di\013eren)o(t)g
(distributed)h(tasks)e(at)h(the)g(same)f(time,)i(and)f(therefore)f(a)h
(1D)f(pro)q(cess)h(grid)h(usually)150 897 y(yields)19
b(the)f(b)q(est)g(p)q(erformance.)28 b(F)l(ortunately)l(,)18
b(most)f(mo)q(dern)h(parallel)h(in)o(terconnection)g(net)o(w)o(orks)150
954 y(are)e(at)g(least)g(as)g(ric)o(h)h(as)f(a)g(2D)g(grid,)h(so)f
(that)g(the)g(additional)i(lev)o(els)f(of)f(parallelism)j(inheren)o(t)e
(in)g(a)150 1010 y(2D)d(pro)q(cess)g(grid)h(can)f(b)q(e)h(successfully)
h(exploited.)221 1068 y(The)e(LU)h(factorization)f(\(used)g(to)f(solv)o
(e)i(a)f(systems)f(of)h(linear)h(equations\))f(can)h(b)q(e)g(used)f(to)
g(illus-)150 1124 y(trate)h(the)i(usefulness)h(of)e(the)g(pro)q(cess)h
(grid.)27 b(Figure)17 b(2)g(sho)o(ws)g(the)h(basic)g(steps)f(of)g(a)g
(righ)o(t-lo)q(oking)150 1181 y(LU)g(factorization)g(as)f(they)h
(a\013ect)f(the)h(matrix's)f(elemen)o(ts.)25 b(The)17
b(\014rst)f(action)h(in)h(the)f(algorithm)f(is)150 1237
y(to)e(form)h(the)g(panel)h(of)e Fp(L)h Fs(as)g(sho)o(wn.)k(A)c(pro)q
(cess)g(column)h(will)h(co)q(op)q(erate)e(to)f(do)h(this.)20
b(This)c(pro)q(cess)150 1294 y(column)f(will)h(then)f(broadcast)f(its)g
(p)q(ortion)h(of)f Fp(L)g Fs(along)g(pro)q(cess)h(ro)o(ws.)j(A)d(pro)q
(cess)f(ro)o(w)g(will)i(use)f(this)150 1350 y(information)g(and)g(co)q
(op)q(erate)f(to)g(form)g Fp(U)5 b Fs(.)20 b Fp(U)f Fs(is)c(then)g
(broadcast)f(within)i(pro)q(cess)f(columns,)g(and)g(all)150
1406 y(pro)q(cesses)h(will)g(use)g(the)f(v)m(alues)i(of)d
Fp(L)h Fs(and)h Fp(U)k Fs(to)15 b(\014nd)1120 1395 y(~)1108
1406 y Fp(A)q Fs(.)842 1509 y Fh(@)863 1530 y(@)p 842
1883 2 416 v 842 1884 63 2 v 858 1692 a Fz(L)p 842 1468
416 2 v 1257 1530 2 63 v 1032 1516 a(U)p 906 1533 353
2 v 906 1882 2 349 v 1076 1718 a(~)1063 1730 y Fg(A)p
1256 1882 V 906 1884 353 2 v 607 1981 a Fs(Figure)g(2:)20
b(After)15 b(\014rst)g(step)g(of)g(LU)g(factorization)221
2089 y(This)20 b(v)o(ery)g(sk)o(etc)o(h)o(y)f(description)i(of)f(LU)g
(is)g(analyzed)h(m)o(uc)o(h)f(more)g(completely)h(in)f([9],)g(whic)o(h)
150 2146 y(includes)d(an)f(examination)f(of)g(scalabilit)o(y)i(and)f
(the)f(adv)m(an)o(tages)f(of)h(using)h(2D)f(pro)q(cess)g(grids.)221
2203 y(A)e(more)g(detailed)j(understanding)e(of)g(the)f(logical)i(pro)q
(cess)f(grid)g(will)i(b)q(e)e(obtained)g(as)g(w)o(e)f(discuss)150
2260 y(the)i(v)m(arious)h(BLA)o(CS)g(routines)f(later)g(in)h(the)g(pap)
q(er.)150 2388 y Fk(4.3)56 b(Con)n(texts)150 2477 y Fs(In)20
b(the)f(BLA)o(CS,)g(eac)o(h)h(logical)g(pro)q(cess)g(grid)f
(\(hereafter)g(referred)g(to)g(simply)h(as)f(the)g(grid\))h(is)f(en-)
150 2533 y(closed)i(in)g(a)e(con)o(text.)34 b(A)20 b(con)o(text)f(ma)o
(y)h(b)q(e)g(though)o(t)g(of)f(as)h(a)f(message)h(passing)g(univ)o
(erse.)35 b(This)150 2589 y(means)19 b(that)g(a)g(grid)h(can)f(safely)h
(comm)o(unicate)g(ev)o(en)f(if)h(other)f(\(p)q(ossibly)i(o)o(v)o
(erlapping\))f(grids)f(are)150 2646 y(also)c(comm)o(unicating.)221
2704 y(In)c(most)f(resp)q(ects,)i(w)o(e)f(can)g(use)g(the)h(terms)e
(`grid')h(and)g(`con)o(text')e(in)o(terc)o(hangeably)l(.)20
b(F)l(or)10 b(example,)1039 2828 y(5)p eop
%%Page: 6 10
6 9 bop 150 199 a Fs(w)o(e)16 b(ma)o(y)f(sa)o(y)g(\\p)q(erform)h(op)q
(eration)g(in)h(con)o(text)e(X")h(or)g(\\in)g(grid)h(X".)e(The)h(sligh)
o(t)h(di\013erence)g(here)f(is)150 256 y(that)i(the)h(user)g(ma)o(y)g
(de\014ne)h(t)o(w)o(o)d(exactly)j(iden)o(tical)g(grids)g(\(sa)o(y)l(,)e
(t)o(w)o(o)g(1x3)g(pro)q(cess)i(grids,)f(b)q(oth)g(of)150
312 y(whic)o(h)i(use)g(pro)q(cesses)g(0,)g(1,)g(and)f(2\),)h(but)g(eac)
o(h)f(will)i(b)q(e)f(wrapp)q(ed)g(in)g(its)g(o)o(wn)f(con)o(text,)h(so)
f(that)150 369 y(they)d(are)f(distinct)i(in)g(op)q(eration,)f(ev)o(en)g
(though)g(they)g(are)f(indistinguishabl)q(e)k(from)c(a)h(pro)q(cess)g
(grid)150 425 y(standp)q(oin)o(t.)221 482 y(Con)o(texts)i(are)h(used)g
(so)g(that)g(individual)j(routines)e(using)f(the)h(BLA)o(CS)f(can,)i
(when)e(required,)150 538 y(safely)c(op)q(erate)f(without)h(w)o
(orrying)f(if)h(the)g(user)g(is)g(running)g(other)g(distributed)h(co)q
(des)f(on)g(the)f(same)150 595 y(mac)o(hine.)221 651
y(Another)k(example)i(of)f(the)g(use)g(of)f(con)o(text)h(migh)o(t)g(b)q
(e)g(to)g(de\014ne)h(a)e(normal)h(2D)f(pro)q(cess)i(grid)150
707 y(within)f(whic)o(h)g(most)e(computation)h(tak)o(es)f(place.)33
b(Ho)o(w)o(ev)o(er,)18 b(in)i(certain)f(p)q(ortions)g(of)g(the)g(co)q
(de)h(it)150 764 y(ma)o(y)13 b(b)q(e)i(more)e(con)o(v)o(enien)o(t)h(to)
f(access)h(the)g(pro)q(cesses)g(as)g(a)f(1D)g(grid,)h(and)g(at)g
(certain)g(other)f(times)h(w)o(e)150 820 y(ma)o(y)j(wish,)i(for)e
(instance,)i(to)e(share)h(information)g(among)f(nearest)h(neigh)o(b)q
(ors.)29 b(W)l(e)18 b(will)h(therefore)150 877 y(w)o(an)o(t)c(eac)o(h)i
(pro)q(cess)f(to)g(ha)o(v)o(e)g(access)g(to)g(three)g(con)o(texts:)22
b(the)16 b(2D)g(grid,)g(the)h(1D)e(grid,)i(and)f(a)g(small)150
933 y(grid)g(whic)o(h)g(con)o(tains)f(the)g(pro)q(cess)h(and)f(its)g
(nearest)g(neigh)o(b)q(ors.)221 990 y(Therefore,)f(w)o(e)h(see)h(that)e
(con)o(text)h(allo)o(ws)g(us)g(to:)218 1084 y Fn(\017)23
b Fs(Create)14 b(arbitrary)h(groups)g(of)f(pro)q(cesses,)218
1178 y Fn(\017)23 b Fs(Create)14 b(an)h(indeterminate)i(n)o(um)o(b)q
(er)e(of)g(o)o(v)o(erlapping)h(and/or)f(disjoin)o(t)h(grids,)218
1272 y Fn(\017)23 b Fs(Isolate)15 b(eac)o(h)g(pro)q(cess)h(grid)f(so)g
(that)g(grids)g(do)g(not)g(in)o(terfere)h(with)f(eac)o(h)g(other.)221
1366 y(In)10 b(the)h(BLA)o(CS,)f(there)g(are)g(t)o(w)o(o)f(grid)h
(creation)h(routines)f(\()p Fl(BLACS)p 1351 1366 15 2
v 16 w(GRIDINIT)f Fs(and)i Fl(BLACS)p 1772 1366 V 16
w(GRIDMAP)p Fs(\))150 1423 y(whic)o(h)18 b(create)e(a)h(pro)q(cess)g
(grid)g(and)g(its)g(enclosing)i(con)o(text.)24 b(These)17
b(routines)h(return)e(con)o(text)h(han-)150 1479 y(dles,)d(whic)o(h)g
(are)f(simple)i(in)o(tegers,)e(assigned)h(b)o(y)f(the)g(BLA)o(CS)h(to)f
(iden)o(tify)h(the)f(con)o(text.)19 b(Subsequen)o(t)150
1536 y(BLA)o(CS)e(routines)g(will)i(b)q(e)e(passed)g(these)g(handles,)h
(whic)o(h)g(allo)o(w)f(the)g(BLA)o(CS)g(to)f(determine)i(from)150
1592 y(whic)o(h)h(con)o(text/grid)g(a)f(routine)h(is)g(b)q(eing)h
(called.)32 b(The)18 b(user)h(should)h(nev)o(er)e(actually)i
(manipulate)150 1649 y(these)h(handles;)j(they)d(are)g(opaque)g(data)f
(ob)s(jects)g(whic)o(h)h(are)g(only)g(meaningful)h(for)f(the)f(BLA)o
(CS)150 1705 y(routines.)221 1762 y(A)i(de\014ned)i(con)o(text)e
(consumes)h(resources.)42 b(It)22 b(is)h(therefore)g(advisable)h(to)d
(release)j(con)o(texts)150 1818 y(when)17 b(they)f(are)g(no)g(longer)g
(needed.)25 b(This)16 b(is)h(done)g(via)f(the)g(routine)h
Fl(BLACS)p 1520 1818 V 17 w(GRIDEXIT)p Fs(.)d(When)j(the)150
1874 y(en)o(tire)e(BLA)o(CS)g(system)f(is)h(sh)o(ut)f(do)o(wn)g(\(via)h
(a)f(call)h(to)f Fl(BLACS)p 1245 1874 V 16 w(EXIT)p Fs(\),)g(all)h
(outstanding)f(con)o(texts)g(are)150 1931 y(automatically)i(freed.)150
2053 y Fk(4.4)56 b(ID-less)17 b(Comm)n(unication)150
2139 y Fs(One)c(of)f(the)g(features)g(that)g(sets)g(the)g(BLA)o(CS)h
(apart)e(from)h(other)g(message)g(passing)g(la)o(y)o(ers)g(is)h(that)f
(the)150 2195 y(user)f(do)q(es)h(not)f(need)h(to)f(sp)q(ecify)i
Fm(message)f(IDs)p Fs(,)f(\(abbreviated)g(msgid\).)19
b(A)11 b(msgid)h(\(also)f(referred)h(to)e(as)150 2252
y(a)15 b(message)f(t)o(yp)q(e)h(or)f(tag\))g(is)h(usually)i(an)e(in)o
(teger)g(whic)o(h)g(allo)o(ws)g(a)g(receiving)h(pro)q(cess)g(to)e
(distinguish)150 2308 y(b)q(et)o(w)o(een)22 b(incoming)i(messages.)40
b(The)22 b(generation)g(of)g(these)g(IDs)g(can)h(b)q(ecome)f
(problematic.)42 b(A)150 2365 y(common)20 b(mistak)o(e)h(is)g(to)f(use)
h(a)f(constan)o(t)g(msgid)h(within)g(a)g(lo)q(op,)h(so)e(that)g(if)h
(one)g(pro)q(cess)g(tak)o(es)150 2421 y(longer)14 b(than)g(others)g(to)
f(\014nish)i(the)f(lo)q(op,)h(it)f(ma)o(y)f(wind)i(up)f(receiving)i
(data)d(from)h(the)g(next)g(iteration)150 2478 y(as)e(this)h
(iteration's)g(data.)18 b(This)13 b(is)h(just)e(the)h(most)f(ob)o
(vious)g(w)o(a)o(y)g(suc)o(h)h(msgid)g(problems)g(can)g(happ)q(en.)150
2534 y(The)k(same)g(result)g(can)g(o)q(ccur)g(whenev)o(er)g(non-unique)
i(IDs)e(are)g(used)g(in)h(an)o(y)e(t)o(w)o(o)g(sections)h(of)f(co)q(de)
150 2591 y(not)k(separated)f(b)o(y)h(an)g(explicit)i(barrier.)35
b(These)20 b(kinds)h(of)f(programming)f(mistak)o(es)h(can)g(lead)h(to)
150 2647 y(non-deterministic)16 b(co)q(de)d(whic)o(h)h(will)h(\014nish)
g(correctly)e(some)g(of)g(the)g(time,)h(giv)o(e)f(wrong)g(results)g
(some)150 2704 y(of)i(the)g(time,)g(and)h(at)e(other)h(times)g(simply)i
(crash.)1039 2828 y(6)p eop
%%Page: 7 11
7 10 bop 221 199 a Fs(Man)o(y)18 b(parallel)j(pro)s(jects)e(are)g(to)q
(o)g(large)h(for)f(one)g(p)q(erson/team)g(to)g(write.)33
b(This)20 b(means)g(that)150 256 y(msgids)15 b(m)o(ust)g(b)q(e)g(co)q
(ordinated)h(b)q(et)o(w)o(een)f(all)h(routines)f(and)g(all)h(writers)f
(of)f(the)h(pac)o(k)m(age.)20 b(If)15 b(another)150 312
y(routine)j(is)f(added)h(at)e(a)h(later)g(date,)g(care)g(m)o(ust)g(b)q
(e)g(tak)o(en)g(to)f(ensure)i(that)e(the)i(new)f(routine's)g(IDs)150
369 y(do)e(not)g(con\015ict)h(with)g(an)o(y)e(other)h(routine's.)221
425 y(Therefore,)c(to)h(add)f(to)h(the)f(programmabilit)o(y)h(of)f(the)
h(BLA)o(CS,)g(it)g(w)o(as)f(decided)j(that)d(the)g(BLA)o(CS)150
482 y(w)o(ould)19 b(in)o(ternally)h(generate)f(the)f(required)i
(msgids.)31 b(These)19 b(generated)g(IDs)f(had)h(to)f(ha)o(v)o(e)h
(certain)150 538 y(prop)q(erties.)h(First,)14 b(it)g(m)o(ust)f(nev)o
(er)h(b)q(e)h(the)f(case)g(that)f(unrelated)i(messages)e(with)i(the)f
(same)f(destina-)150 594 y(tion)k(w)o(ould)f(get)g(the)h(same)f(ID.)g
(Second,)h(in)h(order)e(to)f(main)o(tain)i(p)q(erformance,)g(the)f(ID)h
(generating)150 651 y(algorithm)f(had)g(to)g(use)g(only)h(lo)q(cal)g
(information:)k(o\013-pro)q(cessor)16 b(memory)f(access)h(could)h(not)f
(b)q(e)g(al-)150 707 y(lo)o(w)o(ed.)27 b(F)l(urther,)18
b(it)f(is)h(necessary)g(to)f(allo)o(w)h(the)f(BLA)o(CS)h(to)f(b)q(e)h
(used)h(in)f(conjunction)g(with)g(other)150 764 y(comm)o(unication)g
(platforms.)25 b(An)17 b(example)h(that)f(o)q(ccurs)g(regularly)h(is)f
(linking)j(a)c(BLA)o(CS)i(pac)o(k)m(age)150 820 y(\(for)c(example,)i
(ScaLAP)l(A)o(CK\))g(with)f(a)g(mac)o(hine)h(sp)q(eci\014c)h(pac)o(k)m
(age.)221 877 y(These)f(goals)g(w)o(ere)g(ac)o(hiev)o(ed)h(b)o(y)f
(placing)i(t)o(w)o(o)c(restrictions)j(on)f(comm)o(unication,)h(and)f
(allo)o(wing)150 933 y(the)h(user)g(to)f(optionally)i(sp)q(ecify)g(the)
f(BLA)o(CS)g(msgid)g(range.)25 b(The)17 b(\014rst)f(restriction)i(on)e
(comm)o(uni-)150 990 y(cation)j(is)g(that)f(a)h(receiving)h(pro)q(cess)
f(m)o(ust)f(kno)o(w)g(the)h(pro)q(cess)g(grid)g(co)q(ordinates)h(of)e
(the)h(sending)150 1046 y(pro)q(cess.)g(Second,)14 b(comm)o(unication)f
(b)q(et)o(w)o(een)f(t)o(w)o(o)f(pro)q(cesses)i(is)g(strictly)f
(ordered.)20 b(This)12 b(means)h(that)150 1103 y(if)h
Fn(f)p Fs(0,)e(0)p Fn(g)h Fs(sends)g(t)o(w)o(o)f(messages)h(to)f
Fn(f)p Fs(0,)h(1)p Fn(g)p Fs(,)f(then)i Fn(f)p Fs(0,)e(1)p
Fn(g)h Fm(must)g Fs(receiv)o(e)h(them)f(in)h(the)f(same)g(order)g(that)
150 1159 y(they)i(w)o(ere)g(sen)o(t.)221 1215 y(Finally)l(,)23
b(in)f(order)e(for)g(the)h(BLA)o(CS)h(to)e(co)q(exist)h(with)g(other)g
(comm)o(unication)g(pac)o(k)m(ages,)h(the)150 1272 y(BLA)o(CS)17
b(allo)o(w)f(the)h(user)f(the)g(option)h(of)e(sp)q(ecifying)k(what)c
(range)h(of)g(msgids)h(the)f(BLA)o(CS)h(can)f(use.)150
1328 y(In)k(this)f(case,)h(it)f(is)h(the)f(user's)f(resp)q(onsibilit)o
(y)k(to)c(ensure)i(that)e(the)h(BLA)o(CS)h(msgid)f(range)g(is)g(not)150
1385 y(used)h(in)h(the)f(co)q(de)g(whic)o(h)h(utilizes)g(the)f(other)g
(comm)o(unication)g(pac)o(k)m(age.)33 b(If)20 b(the)g(user)g(wishes)h
(to)150 1441 y(set)f(the)f(BLA)o(CS)i(msgid)f(range,)g(he)g(ma)o(y)f
(do)h(so)f(b)o(y)h(a)f(call)i(to)e(the)h(supp)q(ort)g(routine)g
Fl(BLACS)p 1852 1441 15 2 v 17 w(SET)p Fs(.)150 1498
y(Note)h(that)g(if)i(the)e(BLA)o(CS)i(in)f(use)g(are)g(written)g(on)f
(top)h(of)f(a)g(message)h(passing)g(system)f(whic)o(h)150
1554 y(nativ)o(ely)14 b(supp)q(orts)g(the)g(con)o(text)e(concept)i
(\(for)f(instance,)h(MPI\),)f(passing)h(a)f(unique)i(system)f(con)o
(text)150 1611 y(for)21 b(the)h(BLA)o(CS)h(to)e(use)h(will)i(ensure)e
(BLA)o(CS)g(comm)o(unication)h(will)g(not)f(in)o(terfere)g(with)g
(other)150 1667 y(comm)o(unication)16 b(pac)o(k)m(ages.)k(In)c(this)f
(case,)g(setting)g(msgid)h(range)f(will)i(not)d(b)q(e)i(required.)150
1787 y Fk(4.5)56 b(Blo)r(c)n(king)17 b(Lev)n(els)150
1873 y Fs(An)11 b(understanding)g(of)f(the)g(lev)o(el)i(of)e
Fm(blo)n(cking)f Fs(is)i(required)g(in)g(order)f(to)g(safely)h(use)f
(an)o(y)g(comm)o(unication)150 1930 y(pac)o(k)m(age.)19
b(A)11 b(comm)o(unication)h(op)q(eration)f(has)g(v)m(arious)g
(resources)h(tied)f(to)g(it,)h(the)f(main)g(suc)o(h)h(resource)150
1986 y(b)q(eing)h(the)g(user's)f(bu\013er.)18 b(Since)c(the)e(bu\013er)
h(is)f(the)g(main)h(resource)f(BLA)o(CS)h(users)f(will)i(b)q(e)f
(concerned)150 2043 y(with,)i(in)h(the)f(follo)o(wing)g(discussion)i(w)
o(e)e(will)h(refer)f(only)h(to)e(the)h(user's)g(bu\013er,)f(instead)i
(of)e(using)i(the)150 2099 y(more)f(general)g(term)g(\\resources".)221
2156 y(The)10 b(blo)q(c)o(king)i(lev)o(el)g(of)e(a)h(routine)g(tells)g
(the)g(user)g(what)f(corresp)q(ondence,)i(if)f(an)o(y)l(,)g(there)g(is)
g(b)q(et)o(w)o(een)150 2212 y(the)k(return)g(from)g(a)g(routine,)g(and)
g(the)h(a)o(v)m(ailabilit)o(y)h(of)e(the)g(bu\013er.)20
b(F)l(or)14 b(example,)i(if)g(the)f(user)g(p)q(osts)150
2268 y(a)i(receiv)o(e,)g(he)h(needs)g(to)e(kno)o(w)g(when)i(the)f(data)
f(he)h(is)h(receiving)g(has)f(actually)h(b)q(een)g(stored)e(in)i(the)
150 2325 y(bu\013er.)221 2381 y(In)j(this)f(pap)q(er,)i(w)o(e)e
(de\014ne)i(three)e(lev)o(els)i(of)e(blo)q(c)o(king:)31
b(non-blo)q(c)o(king,)23 b(lo)q(cally-blo)q(c)o(ki)q(ng,)h(and)150
2438 y(globally-blo)q(c)o(king.)k(These)18 b(lev)o(els)g(are)f
(brie\015y)h(preview)o(ed)g(b)q(elo)o(w.)27 b(App)q(endix)19
b(B)e(pro)o(vides)h(a)f(short)150 2494 y(section)f(explaining)h(eac)o
(h)e(of)g(these)h(lev)o(els)g(in)g(greater)e(detail.)218
2591 y Fn(\017)23 b Fm(Non-blo)n(cking)9 b Fs(comm)o(unication:)19
b(the)11 b(return)g(from)g(the)g(comm)o(unication)h(routine)g(implies)h
(only)264 2647 y(that)f(the)h(message)g(request)g(has)h(b)q(een)g(p)q
(osted.)19 b(It)14 b(is)f(then)h(the)f(user's)g(resp)q(onsibilit)o(y)j
(to)c(prob)q(e)264 2704 y(and)j(th)o(us)g(determine)h(when)g(the)f(op)q
(eration)h(has)f(completed.)1039 2828 y(7)p eop
%%Page: 8 12
8 11 bop 218 199 a Fn(\017)23 b Fm(L)n(o)n(c)n(al)r(ly-blo)n(cking)9
b Fs(comm)o(unication:)18 b(Ma)o(y)11 b(b)q(e)h(applied)i(only)e(to)f
(send)h(op)q(erations,)g(not)f(receiv)o(es.)264 256 y(The)16
b(return)g(from)g(the)g(send)h(implies)i(that)c(the)i(bu\013er)f(is)h
(a)o(v)m(ailable)h(for)d(re-use.)24 b(It)16 b(is)h(further)264
312 y(sp)q(eci\014ed)g(that)e(the)h(send)g(will)h(complete)f
(regardless)g(of)f(whether)h(the)g(corresp)q(onding)g(receiv)o(e)264
369 y(is)f(p)q(osted.)218 469 y Fn(\017)23 b Fm(Glob)n(al)r(ly-blo)n
(cking)d Fs(comm)o(unication:)34 b(The)22 b(return)g(from)f(the)h(op)q
(eration)h(implies)h(that)d(the)264 525 y(bu\013er)14
b(is)g(a)o(v)m(ailable)i(for)d(re-use.)20 b(The)14 b(op)q(eration)g(ma)
o(y)f(not)h(complete)g(unless)h(the)f(complemen)o(t)264
582 y(of)h(the)h(op)q(eration)f(is)h(called)i(\(e.g.,)c(a)h(send)h(ma)o
(y)f(not)g(complete)i(if)f(the)f(corresp)q(onding)i(receiv)o(e)264
638 y(is)e(not)g(p)q(osted\).)221 752 y(The)e(BLA)o(CS)h(pro)o(vide)g
(globally-blo)q(c)o(king)i(p)q(oin)o(t)e(to)e(p)q(oin)o(t)i(receiv)o
(e,)h(broadcasts,)d(and)i(com)o(bines.)150 808 y(The)19
b(BLA)o(CS)h(p)q(oin)o(t)f(to)g(p)q(oin)o(t)g(send)h(is)g(lo)q
(cally-blo)q(c)o(king.)35 b(App)q(endix)21 b(B)e(pro)o(vides)h(the)f
(reasoning)150 865 y(b)q(ehind)e(these)f(c)o(hoices)g(for)e(the)i(BLA)o
(CS)f(blo)q(c)o(king)i(lev)o(els.)150 1017 y Fr(5)69
b(Naming)22 b(Con)n(v)n(en)n(tions)150 1121 y Fs(This)13
b(section)f(giv)o(es)h(the)f(naming)g(con)o(v)o(en)o(tions)g(for)g(eac)
o(h)g(of)g(the)g(four)f(BLA)o(CS)i(routine)g(classi\014cations)150
1178 y(\(p)q(oin)o(t)f(to)f(p)q(oin)o(t)i(comm)o(unication,)g
(broadcast,)e(com)o(bine)i(and)f(supp)q(ort\).)19 b(P)o(oin)o(t)12
b(to)f(p)q(oin)o(t,)i(broadcast)150 1234 y(and)i(com)o(bine)h(are)f
(all)h(t)o(yp)q(ed)g(routines,)f(i.e.,)g(there)g(is)h(a)f(separate)g
(routine)g(for)g(eac)o(h)g(data)g(t)o(yp)q(e.)150 1363
y Fq(P)o(oin)o(t)20 b(to)g(P)o(oin)o(t)f(and)h(Broadcast)h(Routines)46
b Fs(The)18 b(names)f(of)f(the)i(comm)o(unication)g(routines)150
1420 y(follo)o(w)12 b(the)g(template)g Fl(vXXYY2D)p Fs(,)f(where)h(the)
g(letter)g(in)h(the)e Fl(v)h Fs(p)q(osition)h(indicates)g(the)f(data)g
(t)o(yp)q(e)g(b)q(eing)150 1476 y(sen)o(t,)17 b Fl(XX)h
Fs(is)g(replaced)g(to)f(indicate)i(the)f(shap)q(e)g(of)f(the)g(matrix,)
h(and)f(the)h Fl(YY)f Fs(p)q(ositions)h(are)g(used)g(to)150
1533 y(indicate)f(the)e(t)o(yp)q(e)g(of)g(comm)o(unication)h(to)f(p)q
(erform.)k(This)d(is)g(sho)o(wn)f(in)h(T)l(able)g(3.)150
1662 y Fq(Com)o(bines)45 b Fs(The)12 b(general)g(form)f(of)h(the)f
(names)h(for)f(com)o(bines)i(is)f Fl(vGZZZ2D)p Fs(,)f(where)h
Fl(v)g Fs(is)g(the)g(same)f(as)150 1718 y(sho)o(wn)k(in)i(T)l(able)f
(3.)21 b(The)16 b(p)q(osition)g Fl(ZZZ)f Fs(indicates)i(what)e(t)o(yp)q
(e)h(of)f(op)q(eration)h(should)g(b)q(e)h(p)q(erformed)150
1774 y(when)f(sending)g(the)f(data.)20 b(The)15 b(op)q(erations)g
(presen)o(tly)h(supp)q(orted)g(are)f(sho)o(wn)g(on)g(T)l(able)h(4.)150
1903 y Fq(Supp)q(ort)f(Routines)47 b Fs(The)13 b(supp)q(ort)g(routines)
g(serv)o(e)g(man)o(y)g(div)o(erse)g(functions,)h(and)f(th)o(us)g(they)g
(do)150 1960 y(not)h(ha)o(v)o(e)g(a)g(great)g(degree)g(of)g
(standardization.)20 b(All)c(o\016cial)f(BLA)o(CS)g(supp)q(ort)f
(routines)h(\(i.e.,)f(those)150 2016 y(that)g(are)h(guaran)o(teed)g(b)o
(y)g(the)g(standard)g(to)g(exist\))g(ha)o(v)o(e)g(the)g(form)g
Fl(BLACS)p 1484 2016 15 2 v 16 w(<name>)p Fs(.)150 2168
y Fr(6)69 b(P)n(oin)n(t)23 b(T)-6 b(o)23 b(P)n(oin)n(t)f(Comm)n
(unication)150 2274 y Fk(6.1)56 b(Seman)n(tics)150 2363
y Fs(P)o(oin)o(t)14 b(to)f(p)q(oin)o(t)h(comm)o(unication)h(requires)f
(t)o(w)o(o)f(complemen)o(tary)h(op)q(erations.)20 b(The)14
b Fm(send)f Fs(op)q(eration)150 2420 y(pro)q(duces)18
b(a)g(message,)f(whic)o(h)i(is)f(then)g(consumed)g(b)o(y)g(the)g
Fm(r)n(e)n(c)n(eive)e Fs(op)q(eration.)27 b(The)18 b(BLA)o(CS)g(send)
150 2476 y(is)h(de\014ned)g(to)f(b)q(e)h(lo)q(cally-blo)q(c)o(kin)q(g,)
i(and)d(the)h(receiv)o(e)g(is)g(globally-blo)q(c)o(king)i(\(see)d(app)q
(endix)i(B)f(for)150 2533 y(details)d(on)f(blo)q(c)o(king\).)221
2591 y(In)c(addition,)h(the)e(BLA)o(CS)h(sp)q(ecify)h(that)e(p)q(oin)o
(t)h(to)f(p)q(oin)o(t)h(messages)f(b)q(et)o(w)o(een)g(t)o(w)o(o)g(giv)o
(en)h(pro)q(cesses)150 2647 y(will)16 b(b)q(e)e(strictly)h(ordered.)20
b(Therefore,)13 b(if)i(pro)q(cess)f(0)g(sends)g(three)g(messages)g
(\(lab)q(el)h(them)f Fp(A)p Fs(,)g Fp(B)r Fs(,)h(and)150
2704 y Fp(C)s Fs(\))i(to)f(pro)q(cess)i(1,)f(pro)q(cess)h(1)f
Fm(must)g Fs(receiv)o(e)h Fp(A)g Fs(b)q(efore)f(it)h(can)f(receiv)o(e)h
Fp(B)r Fs(,)h(and)e(message)g Fp(C)j Fs(can)d(b)q(e)1039
2828 y(8)p eop
%%Page: 9 13
9 12 bop 915 331 a Fk(vXXYY2D)p 408 406 1285 2 v 408
416 V 407 472 2 57 v 416 472 V 441 455 a Fq(v)p 544 472
V 101 w Fs(MEANING)p 1683 472 V 1692 472 V 408 474 1285
2 v 408 484 V 407 541 2 57 v 416 541 V 441 524 a Fl(I)p
544 541 V 105 w Fs(In)o(teger)15 b(data)f(is)i(to)f(b)q(e)g(comm)o
(unicated.)p 1683 541 V 1692 541 V 408 542 1285 2 v 407
599 2 57 v 416 599 V 441 582 a Fl(S)p 544 599 V 105 w
Fs(Single)h(precision)h(real)f(data)e(is)i(to)f(b)q(e)g(comm)o
(unicated.)p 1683 599 V 1692 599 V 408 600 1285 2 v 407
657 2 57 v 416 657 V 441 640 a Fl(D)p 544 657 V 105 w
Fs(Double)h(precision)g(real)g(data)e(is)i(to)f(b)q(e)g(comm)o
(unicated.)p 1683 657 V 1692 657 V 408 658 1285 2 v 407
715 2 57 v 416 715 V 441 698 a Fl(C)p 544 715 V 105 w
Fs(Single)h(precision)h(complex)f(data)f(is)g(to)g(b)q(e)h(comm)o
(unicated.)p 1683 715 V 1692 715 V 408 717 1285 2 v 407
773 2 57 v 416 773 V 441 756 a Fl(Z)p 544 773 V 105 w
Fs(Double)g(precision)g(complex)g(data)f(is)h(to)e(b)q(e)i(comm)o
(unicated.)p 1683 773 V 1692 773 V 408 775 1285 2 v 408
785 V 408 843 V 408 853 V 407 909 2 57 v 416 909 V 441
892 a Fq(XX)p 544 909 V 49 w Fs(MEANING)p 1683 909 V
1692 909 V 408 911 1285 2 v 408 921 V 407 977 2 57 v
416 977 V 441 960 a Fl(GE)p 544 977 V 81 w Fs(The)f(data)g(to)f(b)q(e)i
(comm)o(unicated)g(is)f(stored)g(in)h(a)f(general)p 1683
977 V 1692 977 V 407 1034 V 416 1034 V 544 1034 V 570
1017 a(rectangular)g(matrix.)p 1683 1034 V 1692 1034
V 408 1035 1285 2 v 407 1092 2 57 v 416 1092 V 441 1075
a Fl(TR)p 544 1092 V 81 w Fs(The)g(data)g(to)f(b)q(e)i(comm)o(unicated)
g(is)f(stored)g(in)h(a)p 1683 1092 V 1692 1092 V 407
1148 V 416 1148 V 544 1148 V 570 1131 a(trap)q(ezoidal)g(matrix.)p
1683 1148 V 1692 1148 V 408 1150 1285 2 v 408 1160 V
408 1218 V 408 1228 V 407 1284 2 57 v 416 1284 V 441
1267 a Fq(YY)p 544 1284 V 49 w Fs(MEANING)p 1683 1284
V 1692 1284 V 408 1286 1285 2 v 408 1296 V 407 1352 2
57 v 416 1352 V 441 1336 a(SD)p 544 1352 V 69 w(Send.)k(One)c(pro)q
(cess)g(sends)g(to)e(another.)p 1683 1352 V 1692 1352
V 408 1354 1285 2 v 407 1411 2 57 v 416 1411 V 441 1394
a(R)-5 b(V)p 544 1411 V 67 w(Receiv)o(e.)21 b(One)16
b(pro)q(cess)f(receiv)o(es)i(from)d(another.)p 1683 1411
V 1692 1411 V 408 1412 1285 2 v 407 1469 2 57 v 416 1469
V 441 1452 a(BS)p 544 1469 V 72 w(Broadcast/send.)19
b(A)c(pro)q(cess)h(b)q(egins)g(the)f(broadcast)g(of)p
1683 1469 V 1692 1469 V 407 1525 V 416 1525 V 544 1525
V 570 1508 a(data)f(within)j(a)d(scop)q(e.)p 1683 1525
V 1692 1525 V 408 1527 1285 2 v 407 1583 2 57 v 416 1583
V 441 1566 a(BR)p 544 1583 V 64 w(Broadcast/recv.)19
b(A)c(pro)q(cess)g(receiv)o(es)h(and)g(participates)p
1683 1583 V 1692 1583 V 407 1640 V 416 1640 V 544 1640
V 570 1623 a(in)g(the)f(broadcast)f(of)h(data)g(within)h(a)f(scop)q(e.)
p 1683 1640 V 1692 1640 V 408 1641 1285 2 v 408 1651
V 270 1729 a(T)l(able)h(3:)k(V)l(alues)c(and)f(meanings)h(of)f(the)g
(comm)o(unication)h(routines')f(name)h(p)q(ositions)926
2086 y Fk(vGZZZ2D)p 283 2104 1534 2 v 283 2114 V 282
2171 2 57 v 291 2171 V 316 2154 a Fq(ZZZ)p 450 2171 V
64 w Fs(MEANING)p 1808 2171 V 1817 2171 V 283 2172 1534
2 v 283 2182 V 282 2239 2 57 v 291 2239 V 316 2222 a(AMX)p
450 2239 V 50 w(En)o(tries)f(of)g(result)h(matrix)f(will)h(ha)o(v)o(e)f
(the)h(v)m(alue)g(of)f(the)g(greatest)p 1808 2239 V 1817
2239 V 282 2295 V 291 2295 V 450 2295 V 476 2278 a(absolute)h(v)m(alue)
g(found)g(in)g(that)e(p)q(osition.)p 1808 2295 V 1817
2295 V 283 2297 1534 2 v 282 2353 2 57 v 291 2353 V 316
2337 a(AMN)p 450 2353 V 50 w(En)o(tries)h(of)g(result)h(matrix)f(will)h
(ha)o(v)o(e)f(the)h(v)m(alue)g(of)f(the)g(smallest)p
1808 2353 V 1817 2353 V 282 2410 V 291 2410 V 450 2410
V 476 2393 a(absolute)h(v)m(alue)g(found)g(in)g(that)e(p)q(osition.)p
1808 2410 V 1817 2410 V 283 2412 1534 2 v 282 2468 2
57 v 291 2468 V 316 2451 a(SUM)p 450 2468 V 59 w(En)o(tries)h(of)g
(result)h(matrix)f(will)h(ha)o(v)o(e)f(the)h(summation)f(of)f(that)h(p)
q(osition.)p 1808 2468 V 1817 2468 V 283 2470 1534 2
v 283 2480 V 379 2557 a(T)l(able)h(4:)j(V)l(alues)e(and)e(meanings)h
(of)e(com)o(bine)j(routines')e(name)g(p)q(ositions)1039
2828 y(9)p eop
%%Page: 10 14
10 13 bop 150 199 a Fs(receiv)o(ed)16 b(only)g(after)f(b)q(oth)g
Fp(A)g Fs(and)g Fp(B)r Fs(.)21 b(The)15 b(main)h(reason)f(for)f(this)i
(restriction)g(is)f(that)g(it)g(allo)o(ws)h(for)150 256
y(the)f(computation)g(of)g(message)g(iden)o(ti\014ers,)h(as)f(is)h
(discussed)h(in)f(Section)g(4.4.)221 312 y(It)j(should)i(b)q(e)f
(noted,)g(ho)o(w)o(ev)o(er,)g(that)f(messages)g(from)g(di\013eren)o(t)h
(pro)q(cesses)g(are)f(not)g(ordered.)150 369 y(Therefore,)e(if)g(pro)q
(cesses)g(0)p Fp(;)8 b(:)g(:)g(:)d(;)j Fs(3)15 b(send)j(messages)e
Fp(A;)8 b(:)g(:)g(:)d(;)j(D)q Fs(,)15 b(resp)q(ectiv)o(ely)l(,)k(to)d
(pro)q(cess)h(4,)g(pro)q(cess)150 425 y(4)e(ma)o(y)f(receiv)o(e)i
(these)g(messages)f(in)h(an)o(y)e(order)h(that)g(is)h(con)o(v)o(enien)o
(t.)1027 2828 y(10)p eop
%%Page: 11 15
11 14 bop 150 199 a Fk(6.2)56 b(Syn)n(tax)150 290 y Fs(As)15
b(men)o(tioned)h(in)g(Section)g(5,)e(these)h(routines)h(are)f(t)o(yp)q
(e)g(dep)q(enden)o(t,)h(indicated)h(here)e(b)o(y)g(the)g(pre\014x)150
347 y Fl(v)p Fs(.)25 b(The)17 b(matrix)f(t)o(yp)q(e)h(op)q(erated)g(on)
g(b)o(y)f(these)h(routines)h(will)g(therefore)f(v)m(ary)f(with)i
Fl(v)p Fs(,)e(as)h(sho)o(wn)f(in)150 403 y(T)l(able)g(5.)p
467 460 1167 2 v 467 470 V 466 526 2 57 v 475 526 V 500
509 a Fl(v)p 559 526 V 61 w Fs(Data)e(op)q(erated)h(on)g(is)p
1108 526 V 154 w(TYPE)g(declaration)p 1624 526 V 1633
526 V 467 528 1167 2 v 467 538 V 466 594 2 57 v 475 594
V 500 578 a(I)p 559 594 V 69 w(in)o(teger)p 1108 594
V 413 w(INTEGER)p 1624 594 V 1633 594 V 467 596 1167
2 v 466 653 2 57 v 475 653 V 500 636 a(S)p 559 653 V
60 w(single)h(precision)h(real)p 1108 653 V 159 w(REAL)p
1624 653 V 1633 653 V 467 654 1167 2 v 466 711 2 57 v
475 711 V 500 694 a(D)p 559 711 V 50 w(double)f(precision)h(real)p
1108 711 V 139 w(DOUBLE)f(PRECISION)p 1624 711 V 1633
711 V 467 712 1167 2 v 466 769 2 57 v 475 769 V 500 752
a(C)p 559 769 V 52 w(single)g(precision)h(complex)p 1108
769 V 70 w(COMPLEX)p 1624 769 V 1633 769 V 467 770 1167
2 v 466 827 2 57 v 475 827 V 500 810 a(Z)p 559 827 V
57 w(double)f(precision)h(complex)p 1108 827 V 50 w(DOUBLE)f(COMPLEX)p
1624 827 V 1633 827 V 467 829 1167 2 v 467 839 V 611
916 a(T)l(able)g(5:)j(Pre\014x)d(to)e(t)o(yp)q(e)i(declaration)f
(mapping)221 1039 y(With)h(this)g(in)h(mind,)g(the)f(calling)h
(sequences)g(and)f(parameter)f(declarations)i(for)e(these)i(routines)
150 1095 y(are)e(giv)o(en)h(in)h(the)e(follo)o(wing)h(sections.)22
b(Note)15 b(that)g(output)g(parameters)g(are)g(underlined.)24
b(All)16 b(other)150 1152 y(parameters)e(will)j(b)q(e)f(input,)g(and)f
(th)o(us)g(unc)o(hanged)h(on)f(exit)h(from)f(the)g(routine.)150
1287 y Fq(6.2.1)52 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)f(Sends)150
1380 y Fk(vGESD2D\(ICONTXT,)392 b(M,)19 b(N,)f(A,)g(LD)n(A,)h(RDEST,)f
(CDEST)h(\))150 1438 y(vTRSD2D\(ICONTXT,)f(UPLO,)g(DIA)n(G,)h(M,)g(N,)f
(A,)g(LD)n(A,)h(RDEST,)f(CDEST)h(\))150 1557 y Fq(P)o(arameters:)210
1673 y(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570 1730
y(The)f(BLA)o(CS)h(con)o(text)f(handle.)210 1834 y Fq(UPLO)214
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1890 y(Indicates)f(whether)f
(the)f(matrix)h(is)g(upp)q(er)g(\()p Fl(UPLO)23 b(=)h('U')p
Fs(\))13 b(or)g(lo)o(w)o(er)g(\()p Fl(UPLO)23 b(=)570
1947 y('L')p Fs(\))14 b(trap)q(ezoidal.)210 2051 y Fq(DIA)o(G)220
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 2108 y(Indicates)k(whether)f
(the)g(diagonal)h(of)e(the)h(matrix)g(is)h(unit)f(diagonal)h(\()p
Fl(DIAG)570 2164 y(=)k('U')p Fs(\),)19 b(and)h(th)o(us)g(need)h(not)e
(b)q(e)i(comm)o(unicated,)g(or)f(otherwise)g(\()p Fl(DIAG)j(=)570
2221 y('N')p Fs(\).)210 2325 y Fq(M)310 b Fs(\(input\))16
b(INTEGER)570 2382 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(ro)o(ws)f
(to)g(b)q(e)i(sen)o(t.)210 2486 y Fq(N)319 b Fs(\(input\))16
b(INTEGER)570 2543 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(columns)h
(to)e(b)q(e)i(sen)o(t.)210 2647 y Fq(A)320 b Fs(\(input\))16
b(TYPE)f(arra)o(y)e(of)i(dimension)i(\(LD)o(A,)e(N\))570
2704 y(A)g(p)q(oin)o(ter)h(to)e(the)i(b)q(eginning)h(of)e(the)g
(\(sub\)arra)o(y)f(to)g(b)q(e)i(sen)o(t.)1027 2828 y(11)p
eop
%%Page: 12 16
12 15 bop 210 199 a Fq(LD)o(A)250 b Fs(\(input\))16 b(INTEGER)570
256 y(The)e(leading)i(dimension)g(of)d(the)h(matrix)g(A,)g(i.e.,)g(the)
g(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)570 312 y(successiv)o(e)i
(elemen)o(ts)g(in)g(a)f(matrix)g(ro)o(w.)210 402 y Fq(RDEST)182
b Fs(\(input\))16 b(INTEGER)570 459 y(Pro)q(cess)f(ro)o(w)f(co)q
(ordinate)i(of)f(the)g(destination)h(pro)q(cess.)210
549 y Fq(CDEST)183 b Fs(\(input\))16 b(INTEGER)570 606
y(Pro)q(cess)f(column)h(co)q(ordinate)g(of)f(the)g(destination)h(pro)q
(cess.)150 837 y Fq(6.2.2)52 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)f
(Receiv)o(es)150 924 y Fk(vGER)-6 b(V2D\(ICONTXT,)388
b(M,)19 b(N,)f(A)p 1269 931 43 2 v(,)g(LD)n(A,)h(RSR)n(C,)g(CSR)n(C)h
(\))150 983 y(vTRR)-6 b(V2D\(ICONTXT,)18 b(UPLO,)g(DIA)n(G,)d(M,)k(N,)f
(A)p 1269 990 V(,)g(LD)n(A,)h(RSR)n(C,)g(CSR)n(C)h(\))150
1135 y Fq(P)o(arameters:)210 1247 y(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 1304 y(The)f(BLA)o(CS)h(con)o(text)f(handle.)210
1394 y Fq(UPLO)214 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
1451 y(Indicates)f(whether)f(the)f(matrix)h(is)g(upp)q(er)g(\()p
Fl(UPLO)23 b(=)h('U')p Fs(\))13 b(or)g(lo)o(w)o(er)g(\()p
Fl(UPLO)23 b(=)570 1507 y('L')p Fs(\))14 b(trap)q(ezoidal.)210
1597 y Fq(DIA)o(G)220 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
1654 y(Indicates)k(whether)f(the)g(diagonal)h(of)e(the)h(matrix)g(is)h
(unit)f(diagonal)h(\()p Fl(DIAG)570 1710 y(=)k('U')p
Fs(\),)19 b(and)h(th)o(us)g(need)h(not)e(b)q(e)i(comm)o(unicated,)g(or)
f(otherwise)g(\()p Fl(DIAG)j(=)570 1767 y('N')p Fs(\).)210
1857 y Fq(M)310 b Fs(\(input\))16 b(INTEGER)570 1913
y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(ro)o(ws)f(to)g(b)q(e)i(receiv)o
(ed.)210 2004 y Fq(N)319 b Fs(\(input\))16 b(INTEGER)570
2060 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(columns)h(to)e(b)q(e)i
(receiv)o(ed.)210 2150 y Fq(A)320 b Fs(\(output\))14
b(TYPE)h(arra)o(y)f(\(LD)o(A,)h(N\))570 2207 y(A)g(p)q(oin)o(ter)h(to)e
(the)i(b)q(eginning)h(of)e(the)g(\(sub\)arra)o(y)f(to)g(b)q(e)i(receiv)
o(ed.)210 2297 y Fq(LD)o(A)250 b Fs(\(input\))16 b(INTEGER)570
2354 y(The)e(leading)i(dimension)g(of)d(the)h(matrix)g(A,)g(i.e.,)g
(the)g(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)570 2410
y(successiv)o(e)i(elemen)o(ts)g(in)g(a)f(matrix)g(ro)o(w.)210
2500 y Fq(RSR)o(C)216 b Fs(\(input\))16 b(INTEGER)570
2557 y(Pro)q(cess)f(ro)o(w)f(co)q(ordinate)i(of)f(the)g(source)g(of)g
(the)g(message.)210 2647 y Fq(CSR)o(C)217 b Fs(\(input\))16
b(INTEGER)570 2704 y(Pro)q(cess)f(column)h(co)q(ordinate)g(of)f(the)g
(source)g(of)g(the)g(message.)1027 2828 y(12)p eop
%%Page: 13 17
13 16 bop 150 390 a Fk(6.3)56 b(Example)150 480 y Fs(F)l(or)14
b(a)f(simple)j(example)f(of)f(using)h(BLA)o(CS)f(p)q(oin)o(t)h(to)e(p)q
(oin)o(t)i(message)f(passing,)g(w)o(e)g(sho)o(w)f(co)q(de)i(whic)o(h)
150 536 y(has)g(t)o(w)o(o)f(pro)q(cesses)i(sw)o(ap)e(their)i(copies)g
(of)f(a)g(5)f(elemen)o(t)i(double)h(precision)g(v)o(ector)d
Fl(X)p Fs(.)436 655 y Fl(CALL)24 b(BLACS_GRIDINFO\(ICONTXT,)c(NPROW,)j
(NPCOL,)g(MYPROW,)g(MYPCOL\))436 768 y(IF)h(\(MYPROW.EQ.0)e(.AND.)h
(MYPCOL.EQ.0\))g(THEN)508 824 y(CALL)g(DGESD2D\(ICONTXT,)f(5,)i(1,)f
(X,)h(5,)f(1,)h(0\))508 881 y(CALL)f(DGERV2D\(ICONTXT,)f(5,)i(1,)f(X,)h
(5,)f(1,)h(0\))436 937 y(ELSE)g(IF)f(\(MYPROW.EQ.1)f(.AND.)i
(MYPCOL.EQ.0\))e(THEN)508 994 y(CALL)h(DGESD2D\(ICONTXT,)f(5,)i(1,)f
(X,)h(5,)f(0,)h(0\))508 1050 y(CALL)f(DGERV2D\(ICONTXT,)f(5,)i(1,)f(X,)
h(5,)f(0,)h(0\))436 1107 y(END)g(IF)150 1262 y Fr(7)69
b(Broadcasts)150 1369 y Fk(7.1)56 b(Seman)n(tics)150
1459 y Fs(A)13 b(broadcast)e(sends)i(data)f(p)q(ossessed)h(b)o(y)g(one)
g(pro)q(cess)f(to)g(all)i(pro)q(cesses)f(within)g(a)g(scop)q(e.)19
b(Broadcast,)150 1516 y(m)o(uc)o(h)d(lik)o(e)h(p)q(oin)o(t)g(to)e(p)q
(oin)o(t)i(comm)o(unication,)g(has)f(t)o(w)o(o)e(complemen)o(tary)j(op)
q(erations.)22 b(The)17 b(pro)q(cess)150 1572 y(that)e(o)o(wns)f(the)i
(data)f(to)f(b)q(e)i(broadcast)f(issues)h(a)f Fm(br)n(o)n(adc)n
(ast/send)p Fs(.)20 b(All)c(pro)q(cesses)g(within)h(the)e(same)150
1629 y(scop)q(e)h(m)o(ust)e(then)i(issue)g(the)f(complemen)o(tary)h
Fm(br)n(o)n(adc)n(ast/r)n(e)n(c)n(eive)p Fs(.)221 1687
y(The)10 b(BLA)o(CS)h(de\014ne)g(that)e(b)q(oth)i(broadcast/send)e(and)
i(broadcast/receiv)o(e)f(are)g(globally-blo)q(c)o(king)150
1744 y(\(see)20 b(app)q(endix)h(B)f(for)f(details\).)34
b(This)21 b(has)e(sev)o(eral)h(imp)q(ortan)o(t)g(implications.)35
b(The)20 b(\014rst)f(is)i(that)150 1800 y(scop)q(ed)i(op)q(erations)f
(\(broadcasts)f(or)h(com)o(bines\))h(m)o(ust)e(b)q(e)i(strictly)g
(ordered,)h(i.e.,)f(all)h(pro)q(cesses)150 1857 y(within)19
b(a)e(scop)q(e)i(m)o(ust)e(agree)g(on)h(the)g(order)f(of)h(calls)h(to)e
(separate)g(scop)q(ed)i(op)q(erations.)27 b(This)19 b(con-)150
1913 y(strain)o(t)e(falls)h(in)g(line)h(with)e(that)g(already)g(in)h
(place)h(for)d(the)i(computation)f(of)g(message)g(IDs,)g(and)g(is)150
1970 y(presen)o(t)e(in)h(p)q(oin)o(t)g(to)e(p)q(oin)o(t)i(comm)o
(unication)g(as)f(w)o(ell.)221 2028 y(A)j(less)h(ob)o(vious)g(result)g
(is)g(that)e(scop)q(ed)j(op)q(erations)e(with)h Fl(SCOPE)k(=)h('ALL')18
b Fs(m)o(ust)g(b)q(e)h(ordered)150 2085 y(with)14 b(resp)q(ect)h(to)f
(an)o(y)f(other)h(scop)q(ed)h(op)q(eration.)20 b(This)14
b(means)g(that)g(if)g(there)h(are)f(t)o(w)o(o)e(broadcasts)i(to)150
2141 y(b)q(e)g(done,)h(one)f(along)f(a)h(column,)g(and)g(one)g(in)o(v)o
(olving)h(the)f(en)o(tire)h(pro)q(cess)f(grid,)g(all)h(pro)q(cesses)f
(within)150 2197 y(the)19 b(pro)q(cess)g(column)g(issuing)h(the)f
(column)h(broadcast)d(m)o(ust)i(agree)f(on)g(whic)o(h)i(broadcast)e
(will)i(b)q(e)150 2254 y(p)q(erformed)15 b(\014rst.)150
2388 y Fk(7.2)56 b(Syn)n(tax)150 2478 y Fs(As)14 b(with)g(p)q(oin)o(t)g
(to)f(p)q(oin)o(t)h(comm)o(unication,)h(these)e(routines)i(v)m(ary)e
(with)h(the)g(data)f(t)o(yp)q(e,)h(and)g(T)l(able)g(5)150
2534 y(sho)o(ws)22 b(the)g(mapping)h(b)q(et)o(w)o(een)g(the)f(t)o(yp)q
(e)h(pre\014x)f(\(indicated)i(b)q(elo)o(w)f(b)o(y)f Fl(v)p
Fs(\))g(and)h(the)f(data)g(t)o(yp)q(e)150 2591 y(declaration.)34
b(As)19 b(b)q(efore,)i(output)e(parameters)g(are)g(underlined.)36
b(All)21 b(other)e(parameters)g(will)i(b)q(e)150 2647
y(input,)15 b(and)f(th)o(us)f(unc)o(hanged)i(on)e(exit)h(from)f(the)h
(routine.)20 b(With)14 b(these)g(p)q(oin)o(ts)g(in)h(mind,)f(the)g
(calling)150 2704 y(sequences)h(and)g(parameter)e(declarations)i(for)f
(these)h(routines)f(are)g(giv)o(en)h(in)g(the)g(follo)o(wing)g
(sections.)1027 2828 y(13)p eop
%%Page: 14 18
14 17 bop 150 199 a Fq(7.2.1)52 b(Broadcast/send)150
287 y Fk(vGEBS2D\()18 b(ICONTXT,)h(SCOPE,)g(TOP)-5 b(,)393
b(M,)18 b(N,)g(A,)h(LD)n(A)g(\))150 345 y(vTRBS2D\()f(ICONTXT,)h
(SCOPE,)g(TOP)-5 b(,)18 b(UPLO,)h(DIA)n(G,)g(M,)f(N,)g(A,)h(LD)n(A)g
(\))150 451 y Fq(P)o(arameters:)210 564 y(ICONTXT)110
b Fs(\(input\))16 b(INTEGER)570 620 y(The)f(BLA)o(CS)h(con)o(text)f
(handle.)210 714 y Fq(SCOPE)184 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
771 y(Scop)q(e)24 b(of)e(pro)q(cesses)i(to)e(participate)h(in)h(op)q
(eration.)43 b(Limited)25 b(to)d Fl('ROW')p Fs(,)570
827 y Fl('COLUMN')p Fs(,)14 b(or)g Fl('ALL')p Fs(.)g(See)i(Section)g
(4.2)f(for)f(additional)j(details.)210 921 y Fq(TOP)249
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 977 y(Net)o(w)o(ork)f(top)q
(ology)h(to)g(b)q(e)g(em)o(ulated)h(during)g(comm)o(unication.)24
b(T)l(op)q(ologies)570 1034 y(presen)o(tly)16 b(supp)q(orted)g(are)e
(discussed)j(in)f(Section)g(7.4.)210 1128 y Fq(UPLO)214
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1184 y(Indicates)f(whether)f
(the)f(matrix)h(is)g(upp)q(er)g(\()p Fl(UPLO)23 b(=)h('U')p
Fs(\))13 b(or)g(lo)o(w)o(er)g(\()p Fl(UPLO)23 b(=)570
1241 y('L')p Fs(\))14 b(trap)q(ezoidal.)210 1334 y Fq(DIA)o(G)220
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1391 y(Indicates)k(whether)f
(the)g(diagonal)h(of)e(the)h(matrix)g(is)h(unit)f(diagonal)h(\()p
Fl(DIAG)570 1447 y(=)k('U')p Fs(\),)19 b(and)h(th)o(us)g(need)h(not)e
(b)q(e)i(comm)o(unicated,)g(or)f(otherwise)g(\()p Fl(DIAG)j(=)570
1504 y('N')p Fs(\).)210 1598 y Fq(M)310 b Fs(\(input\))16
b(INTEGER)570 1654 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(ro)o(ws)f
(to)g(b)q(e)i(broadcast.)210 1748 y Fq(N)319 b Fs(\(input\))16
b(INTEGER)570 1804 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(columns)h
(to)e(b)q(e)i(broadcast.)210 1898 y Fq(A)320 b Fs(\(input\))16
b(TYPE)f(arra)o(y)e(\(LD)o(A,)i(N\))570 1955 y(A)g(p)q(oin)o(ter)h(to)e
(the)i(b)q(eginning)h(of)e(the)g(\(sub\)arra)o(y)f(to)g(b)q(e)i
(broadcast.)210 2048 y Fq(LD)o(A)250 b Fs(\(input\))16
b(INTEGER)570 2105 y(The)e(leading)i(dimension)g(of)d(the)h(matrix)g
(A,)g(i.e.,)g(the)g(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)570
2161 y(successiv)o(e)i(elemen)o(ts)g(in)g(a)f(matrix)g(ro)o(w.)1027
2828 y(14)p eop
%%Page: 15 19
15 18 bop 150 199 a Fq(7.2.2)52 b(Broadcast/receiv)o(e)150
287 y Fk(vGEBR2D\()17 b(ICONTXT,)i(SCOPE,)g(TOP)-5 b(,)393
b(M,)19 b(N,)f(A)p 1667 294 43 2 v(,)g(LD)n(A,)453 345
y(RSR)n(C,)h(CSR)n(C)h(\))150 403 y(vTRBR2D\()d(ICONTXT,)i(SCOPE,)g
(TOP)-5 b(,)19 b(UPLO,)f(DIA)n(G,)h(M,)g(N,)f(A)p 1667
410 V(,)g(LD)n(A,)453 461 y(RSR)n(C,)h(CSR)n(C)h(\))150
614 y Fq(P)o(arameters:)210 727 y(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 783 y(The)f(BLA)o(CS)h(con)o(text)f(handle.)210
873 y Fq(SCOPE)184 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
930 y(Scop)q(e)24 b(of)e(pro)q(cesses)i(to)e(participate)h(in)h(op)q
(eration.)43 b(Limited)25 b(to)d Fl('ROW')p Fs(,)570
986 y Fl('COLUMN')p Fs(,)14 b(or)g Fl('ALL')p Fs(.)g(See)i(Section)g
(4.2)f(for)f(additional)j(details.)210 1077 y Fq(TOP)249
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1133 y(Net)o(w)o(ork)f(top)q
(ology)h(to)g(b)q(e)g(em)o(ulated)h(during)g(comm)o(unication.)24
b(T)l(op)q(ologies)570 1190 y(presen)o(tly)16 b(supp)q(orted)g(are)e
(discussed)j(in)f(Section)g(7.4.)210 1280 y Fq(UPLO)214
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1337 y(Indicates)f(whether)f
(the)f(matrix)h(is)g(upp)q(er)g(\()p Fl(UPLO)23 b(=)h('U')p
Fs(\))13 b(or)g(lo)o(w)o(er)g(\()p Fl(UPLO)23 b(=)570
1393 y('L')p Fs(\))14 b(trap)q(ezoidal.)210 1483 y Fq(DIA)o(G)220
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 1540 y(Indicates)k(whether)f
(the)g(diagonal)h(of)e(the)h(matrix)g(is)h(unit)f(diagonal)h(\()p
Fl(DIAG)570 1596 y(=)k('U')p Fs(\),)19 b(and)h(th)o(us)g(need)h(not)e
(b)q(e)i(comm)o(unicated,)g(or)f(otherwise)g(\()p Fl(DIAG)j(=)570
1653 y('N')p Fs(\).)210 1743 y Fq(M)310 b Fs(\(input\))16
b(INTEGER)570 1800 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(ro)o(ws)f
(to)g(b)q(e)i(broadcast.)210 1890 y Fq(N)319 b Fs(\(input\))16
b(INTEGER)570 1947 y(The)f(n)o(um)o(b)q(er)h(of)f(matrix)g(columns)h
(to)e(b)q(e)i(broadcast.)210 2037 y Fq(A)320 b Fs(\(output\))14
b(TYPE)h(arra)o(y)f(\(LD)o(A,)h(N\))570 2094 y(A)e(p)q(oin)o(ter)g(to)f
(the)h(b)q(eginning)i(of)d(the)h(\(sub\)arra)o(y)e(to)h(b)q(e)h(receiv)
o(ed/broadcast.)210 2184 y Fq(LD)o(A)250 b Fs(\(input\))16
b(INTEGER)570 2240 y(The)e(leading)i(dimension)g(of)d(the)h(matrix)g
(A,)g(i.e.,)g(the)g(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)570
2297 y(successiv)o(e)i(elemen)o(ts)g(in)g(a)f(matrix)g(ro)o(w.)210
2387 y Fq(RSR)o(C)216 b Fs(\(input\))16 b(INTEGER)570
2444 y(Pro)q(cess)f(ro)o(w)f(co)q(ordinate)i(of)f(the)g(source)g(of)g
(the)g(broadcast.)210 2534 y Fq(CSR)o(C)217 b Fs(\(input\))16
b(INTEGER)570 2591 y(Pro)q(cess)f(column)h(co)q(ordinate)g(of)f(the)g
(source)g(of)g(the)g(broadcast.)1027 2828 y(15)p eop
%%Page: 16 20
16 19 bop 150 199 a Fk(7.3)56 b(Example)150 287 y Fs(As)13
b(describ)q(ed)j(ab)q(o)o(v)o(e,)d(the)h(parameters)e
Fl(M)p Fs(,)h Fl(N)p Fs(,)g(and)h Fl(LDA)f Fs(dictate)h(the)f(shap)q(e)
h(of)f(the)h(arra)o(y)e(b)q(eing)j(com-)150 344 y(m)o(unicated.)21
b(All)16 b(pro)q(cesses)f(participating)h(in)g(a)e(giv)o(en)i(send)f
(op)q(eration)g(or)g(its)g(receiv)o(e)g(complemen)o(t)150
400 y(m)o(ust)g(ha)o(v)o(e)g(the)h(same)g(amoun)o(t)f(of)g(arra)o(y)f
(space)i(a)o(v)m(ailable)i(\(i.e.)j Fl(M)j(*)g(N)15 b
Fs(m)o(ust)g(b)q(e)i(the)f(same\).)k(Ho)o(w-)150 456
y(ev)o(er,)15 b(it)h(is)h(not)e(necessary)h(that)f(they)h(all)g(receiv)
o(e)h(the)f(data)f(in)i(the)e(same)h(w)o(a)o(y)f(\(this)g(holds)i(true)
f(for)150 513 y(p)q(oin)o(t)g(to)e(p)q(oin)o(t)i(comm)o(unication,)f
(as)g(w)o(ell\).)21 b(An)15 b(example)h(should)h(help)f(illustrate)g
(this)g(principle:)221 570 y(Pro)q(cess)h Fn(f)p Fs(0,2)p
Fn(g)f Fs(has)h(a)g(double)i(precision)g(matrix)e Fl(B)p
Fs(,)g(with)h(a)f(total)g(size)h(of)f(500)g(x)g(200.)26
b(All)19 b(the)150 627 y(other)f(pro)q(cesses)h(in)h(its)f(pro)q(cess)g
(column)g(require)h(\014v)o(e)f(ro)o(ws)e(and)i(sev)o(en)g(columns)h
(of)e(this)h(matrix)150 683 y(starting)e(at)f(the)h(matrix)g(p)q
(osition)i(\(9,4\).)24 b(It)17 b(is)h(not)f(necessary)g(for)g(all)h
(participating)g(pro)q(cesses)g(to)150 740 y(receiv)o(e)k(the)f(matrix)
g(in)h(the)f(same)f(w)o(a)o(y)l(.)37 b(F)l(or)20 b(instance,)j(pro)q
(cess)e Fn(f)p Fs(1,2)p Fn(g)f Fs(migh)o(t)h(w)o(an)o(t)f(to)g(receiv)o
(e)150 796 y(the)c(information)h(in)o(to)f(a)g(w)o(ork)f(v)o(ector,)h
Fl(WORK)p Fs(,)f(while)j(the)e(other)g(pro)q(cesses)h(in)g(the)g(pro)q
(cess)f(column)150 853 y(receiv)o(e)g(the)f(the)h(broadcast)e(in)o(to)h
(their)h(cop)o(y)f(of)g Fl(B)p Fs(.)f(This)i(could)g(b)q(e)g
(accomplished)h(as)e(follo)o(ws:)245 965 y Fl(CALL)24
b(BLACS_GRIDINIT\(ICONTXT,)c(NPROW,)k(NPCOL,)f(MYPROW,)f(MYPCOL\))150
1022 y(*)150 1078 y(*)71 b(If)24 b(I)g(participate)e(in)i(the)f
(broadcast)150 1135 y(*)245 1191 y(IF)h(\(MYPCOL)f(.EQ.)g(2\))h(THEN)
150 1247 y(*)150 1304 y(*)143 b(If)24 b(I'm)f(the)h(source)f(of)g(the)h
(broadcast)150 1360 y(*)317 1417 y(IF)g(\(MYPROW)f(.EQ.)g(0\))g(THEN)
389 1473 y(CALL)g(DGEBS2D\(ICONTXT,)f('COLUMN',)g(')i(',)g(5,)f(7,)h
(B\(9,4\),)f(500\))150 1530 y(*)150 1586 y(*)143 b(If)24
b(I)f(want)h(to)f(receive)g(into)g(work)150 1643 y(*)317
1699 y(ELSE)g(IF)h(\(MYPROW)f(.EQ.)g(1\))h(THEN)389 1756
y(CALL)f(DGEBR2D\(ICONTXT,)f('COLUMN,)g(')i(',)g(5,)f(7,)h(WORK,)f(5,)h
(0,)f(2\))150 1812 y(*)150 1868 y(*)143 b(If)24 b(I)f(want)h(to)f
(receive)g(into)g(B)150 1925 y(*)317 1981 y(ELSE)389
2038 y(CALL)g(DGEBR2D\(ICONTXT,)f('COLUMN',)g(')i(',)g(5,)f(7,)h
(B\(9,4\),)f(500,)g(0,)h(2\))317 2094 y(END)f(IF)221
2206 y Fs(NOTE:)14 b(All)i(v)o(ersions)f(of)g(the)g(BLA)o(CS)g(except)g
(PVM)g(allo)o(w)g(the)f(user)h(to)g(v)m(ary)f Fl(M)h
Fs(and)g Fl(N)p Fs(,)f(as)h(long)150 2262 y(as)f Fl(M)24
b(*)g(N)15 b Fs(is)g(the)g(same)f(across)g(all)i(pro)q(cesses.)k(Ho)o
(w)o(ev)o(er,)14 b(in)h(PVM)g(the)g(data)f(m)o(ust)g(b)q(e)i(unpac)o(k)
o(ed)f(in)150 2319 y(the)e(same)g(manner)h(that)e(it)i(is)f(pac)o(k)o
(ed.)20 b(Therefore,)13 b(the)g(shap)q(e)h(of)f(the)g(matrix)g(b)q
(eing)h(comm)o(unicated)150 2375 y(should)i(b)q(e)g(c)o(hanged)g(only)f
(b)o(y)g(v)m(arying)h Fl(LDA)p Fs(.)150 2503 y Fk(7.4)56
b(T)-5 b(op)r(ologies)150 2591 y Fs(The)17 b(top)q(ology)f(parameter)g
(determines)i(ho)o(w)e(the)h(messages)f(in)o(v)o(olv)o(ed)i(in)f(a)g
(distributed)h(op)q(eration)150 2647 y(are)13 b(sen)o(t.)19
b(The)14 b(use)g(of)f(the)g(top)q(ology)g(concept)h(allo)o(ws)g(the)f
(user)h(to)f(exploit)h(the)g(follo)o(wing)g(fact:)19
b(ev)o(en)150 2704 y(if)13 b(the)h(time)f(to)f(p)q(erform)h(a)g
(distributed)h(op)q(eration)f(cannot)g(b)q(e)h(reduced,)g(whic)o(h)g
(pro)q(cessors)f(b)q(ear)g(the)1027 2828 y(16)p eop
%%Page: 17 21
17 20 bop 150 199 a Fs(brun)o(t)17 b(of)g(the)g(cost)g(of)f(the)h(op)q
(eration)h Fm(c)n(an)e Fs(b)q(e)i(v)m(aried.)27 b(T)l(op)q(ology)17
b(also)g(allo)o(ws)g(for)g(the)g(building)i(of)150 256
y(comm)o(unication)d(pip)q(elines,)i(as)d(discussed)h(b)q(elo)o(w.)221
313 y(There)f(are)g(t)o(w)o(o)f(main)h(classes)h(of)f(top)q(ologies)g
(within)i(the)e(BLA)o(CS:)218 411 y Fn(\017)23 b Fs(Pip)q(elining)18
b(top)q(ologies)e(\(ring-based\))218 510 y Fn(\017)23
b Fs(Non-pip)q(elini)q(ng)18 b(top)q(ologies)d(\(tree-based\))221
608 y(In)i(a)f(pip)q(elini)q(ng)k(top)q(ology)l(,)c(the)h(\014rst)f(op)
q(eration)h(sync)o(hronizes)h(the)f(pro)q(cessors)f(so)g(that)g(subse-)
150 664 y(quen)o(t)g(op)q(erations)f(will)j(b)q(e)e(c)o(heap.)21
b(Therefore,)15 b(if)h(the)g(user)g(is)g(a)o(w)o(are)e(that)h(sev)o
(eral)h(broadcasts)e(will)150 721 y(b)q(e)j(p)q(erformed)g(with)g(no)g
(in)o(terlea)o(v)o(ed)h(sync)o(hronization,)f(pip)q(elini)q(ng)j(top)q
(ologies)d(should)g(b)q(e)h(consid-)150 777 y(ered.)26
b(F)l(urther,)18 b(the)f(BLA)o(CS)h(pip)q(elining)i(top)q(ologies)e
(are)f(all)h(based)g(on)f(rings,)g(whic)o(h)i(means)e(that)150
834 y(pip)q(elines)h(can)d(b)q(e)g(main)o(tained)h(if)f(the)g
(algorithm)g(\015o)o(ws)f(across)h(pro)q(cessors)f(in)i(an)e(orderly)i
(w)o(a)o(y)l(.)j(F)l(or)150 890 y(example,)c(if)h(the)f(sender)g(of)g
(ro)o(w)f(broadcasts)g(starts)f(out)i(as)f(the)h(\014rst)g(pro)q(cess)g
(column,)g(and)g(then)g(is)150 946 y(the)i(second,)i(etc.,)e(an)g
(increasing)i(ring)f(pip)q(eline)i(will)f(b)q(e)f(main)o(tained.)28
b(If)18 b(the)f(program)f(\015o)o(w)h(is)h(in)150 1003
y(the)f(opp)q(osite)g(direction,)h(it)f(ma)o(y)f(b)q(e)h(p)q(ossible)i
(to)d(set)g(up)h(a)g(decreasing)g(ring)g(pip)q(eline)q(.)27
b(A)17 b(pip)q(eline)150 1059 y(for)f(increasing)i(direction)g(can)f(b)
q(e)g(obtained)g(b)o(y)g(setting)f Fl(TOP)24 b(=)g('INCREASING)e(RING')
p Fs(;)15 b(a)i(pip)q(eline)150 1116 y(for)e(co)q(des)h(\015o)o(wing)f
(across)g(the)g(pro)q(cessors)g(in)h(the)g(opp)q(osite)g(w)o(a)o(y)e
(can)i(b)q(e)g(obtained)g(b)o(y)f(setting)g Fl(TOP)150
1172 y(=)24 b('DECREASING)e(RING')p Fs(.)221 1230 y(The)15
b(BLA)o(CS')g(pip)q(elinin)q(g)i(top)q(ologies)f(are)e(usually)j(m)o
(uc)o(h)e(slo)o(w)o(er)f(than)h(non-pip)q(elini)q(ng)j(top)q(olo-)150
1286 y(gies)d(if)f(only)h(one)f(op)q(eration)g(is)h(p)q(erformed.)20
b(Pip)q(elining)d(top)q(ologies)e(are)f(used)g(to)g(minimize)i(the)e
(cost)150 1343 y(of)k(sev)o(eral)h(related)g(op)q(erations.)29
b(Therefore,)19 b(if)g(the)f(broadcast)g(do)q(es)h(not)f(pip)q(eline,)k
(the)d(user)f(will)150 1399 y(probably)g(wish)h(to)e(utilize)j(the)e
(top)q(ology)g(whic)o(h)h(minimizes)h(the)e(time)g(sp)q(en)o(t)g(in)h
(only)f(one)g(broad-)150 1456 y(cast.)23 b(The)17 b(BLA)o(CS)g(pro)o
(vide)g(a)f(default)h(top)q(ology)f(whic)o(h)i(attempts)d(to)h(do)h
(this,)f(whic)o(h)i(is)f(in)o(v)o(ok)o(ed)150 1512 y(b)o(y)e(setting)g
Fl(TOP)24 b(=)g(')f(')p Fs(.)221 1570 y(One)16 b(of)g(the)g(three)f
(top)q(ologies)i(ab)q(o)o(v)o(e)e(will)j(probably)e(satisfy)g(most)f
(users.)22 b(Ho)o(w)o(ev)o(er,)14 b(there)i(are)150 1626
y(man)o(y)e(other)h(BLA)o(CS)g(top)q(ologies)g(within)h(the)f(t)o(w)o
(o)e(main)i(classes.)21 b(In)15 b(the)g(pip)q(elini)q(ng)i(category)l
(,)d(the)150 1682 y(user)h(ma)o(y)f(use)i(a)e(split)j(ring)e(\(TOP)g(=)
g('SPLIT)g(RING'\))g(or)g(a)g(m)o(ultiring)h(\(TOP)e(=)i
('Multiring'\),)f(for)150 1739 y(instance.)28 b(There)18
b(are)f(also)g(sev)o(eral)h(t)o(yp)q(es)g(of)f(tree-based)h(top)q
(ologies.)27 b(App)q(endix)20 b(E)e(pro)o(vides)g(full)150
1795 y(details)e(on)f(the)h(a)o(v)m(ailable)g(top)q(ologies.)150
1945 y Fr(8)69 b(Com)n(bines)150 2050 y Fk(8.1)56 b(Seman)n(tics)150
2138 y Fs(In)14 b(a)e(com)o(bine)i(op)q(eration,)f(eac)o(h)g
(participating)h(pro)q(cess)f(con)o(tributes)g(data)g(whic)o(h)g(is)h
(com)o(bined)g(with)150 2194 y(other)k(pro)q(cesses')g(data)f(to)g(pro)
q(duce)i(a)f(result.)29 b(This)19 b(result)f(can)g(b)q(e)h(left)f(on)g
(a)g(particular)h(pro)q(cess)150 2251 y(\(called)h(the)f
Fm(destination)g Fs(pro)q(cess\),)g(or)g(on)g(all)h(participating)g
(pro)q(cesses.)32 b(If)19 b(the)h(result)f(is)h(left)f(on)150
2307 y(only)e(one)g(pro)q(cess,)g(w)o(e)g(refer)f(to)g(the)h(op)q
(eration)g(as)f(a)h Fm(le)n(ave-on-one)f Fs(com)o(bine,)h(and)g(if)g
(the)g(result)g(is)150 2364 y(giv)o(en)f(to)e(all)i(participating)h
(pro)q(cesses)e(w)o(e)g(reference)h(it)g(as)e(a)h Fm(le)n(ave-on-al)r
(l)g Fs(com)o(bine.)221 2421 y(A)o(t)g(presen)o(t,)h(three)g(kinds)g
(of)g(com)o(bines)g(are)g(supp)q(orted.)22 b(They)17
b(are)e(elemen)o(t-wise)i(summation,)150 2478 y(elemen)o(t-wise)d
(absolute)f(v)m(alue)g(maximization,)h(and)f(elemen)o(t-wise)g
(absolute)g(v)m(alue)h(minimization)g(of)150 2534 y(general)d
(rectangular)g(arra)o(ys.)17 b(Note)10 b(that)g(a)g(com)o(bine)h(op)q
(eration)g(com)o(bines)g(data)f(b)q(et)o(w)o(een)h(pro)q(cesses.)150
2591 y(By)h(de\014nition,)j(then,)e(a)f(com)o(bine)h(p)q(erformed)f
(across)g(a)g(scop)q(e)h(of)f(only)g(one)h(pro)q(cess)f(do)q(es)h(not)f
(c)o(hange)150 2647 y(the)j(input)h(data.)j(This)c(is)g(wh)o(y)g(w)o(e)
g(sp)q(ecify)h(that)e(the)h(op)q(erations)g(are)f Fm(element-wise)p
Fs(.)19 b(Elemen)o(t-wise)150 2704 y(indicates)i(that)f(eac)o(h)g
(elemen)o(t)g(of)g(the)g(input)h(arra)o(y)d(will)k(b)q(e)f(com)o(bined)
g(with)f(the)g(corresp)q(onding)1027 2828 y(17)p eop
%%Page: 18 22
18 21 bop 150 199 a Fs(elemen)o(t)14 b(from)f(all)h(other)f(pro)q
(cesses')h(arra)o(ys)e(to)h(pro)q(duce)h(the)g(result.)20
b(Th)o(us,)13 b(a)g(4)7 b Fn(\002)g Fs(2)13 b(arra)o(y)f(of)h(inputs)
150 256 y(pro)q(duces)19 b(a)e(4)11 b Fn(\002)i Fs(2)k(answ)o(er)g
(arra)o(y)l(.)26 b(If)18 b(the)g(elemen)o(t-wise)h(op)q(eration)f
(concept)h(is)f(still)h(unclear,)g(the)150 312 y(examples)d(section)g
(should)g(pro)o(vide)g(further)f(clari\014cation.)221
369 y(The)e(maximization)h(and)f(minimization)i(op)q(erations)e(ma)o(y)
f(require)i(further)e(explanation.)21 b(When)150 425
y(the)e(max/min)g(comparison)g(is)h(b)q(eing)g(p)q(erformed,)g
(absolute)f(v)m(alue)i(is)e(used.)32 b(Therefore,)20
b Fn(\000)p Fs(5)f(and)150 482 y(5)h(are)g(equiv)m(alen)o(t.)38
b(Ho)o(w)o(ev)o(er,)20 b(the)h(returned)g(v)m(alue)g(is)g(unc)o
(hanged;)j(i.e.)36 b(it)21 b(is)g(not)f(the)h(absolute)150
538 y(v)m(alue,)16 b(but)f(instead)h(is)g(the)f(signed)i(v)m(alue.)k
(Therefore,)14 b(if)i(w)o(e)f(p)q(erformed)h(a)e(BLA)o(CS)i(absolute)g
(v)m(alue)150 594 y(maxim)o(um)f(com)o(bine)h(on)f(the)h(n)o(um)o(b)q
(ers)f Fn(\000)p Fs(5)p Fp(;)8 b Fs(3)p Fp(;)g Fs(1)p
Fp(;)g Fn(\000)p Fs(8,)k(the)j(result)h(w)o(ould)g(b)q(e)f
Fn(\000)p Fs(8.)221 651 y(The)g(BLA)o(CS)h(com)o(bines)g(are)f
(globally-blo)q(c)o(king)j(\(see)d(app)q(endix)i(B)e(for)g(details\).)
150 770 y Fk(8.2)56 b(Syn)n(tax)150 856 y Fs(As)14 b(with)g(p)q(oin)o
(t)g(to)f(p)q(oin)o(t)h(comm)o(unication,)h(these)e(routines)i(v)m(ary)
e(with)h(the)g(data)f(t)o(yp)q(e,)h(and)g(T)l(able)g(5)150
913 y(sho)o(ws)22 b(the)g(mapping)h(b)q(et)o(w)o(een)g(the)f(t)o(yp)q
(e)h(pre\014x)f(\(indicated)i(b)q(elo)o(w)f(b)o(y)f Fl(v)p
Fs(\))g(and)h(the)f(data)g(t)o(yp)q(e)150 969 y(declaration.)34
b(As)19 b(b)q(efore,)i(output)e(parameters)g(are)g(underlined.)36
b(All)21 b(other)e(parameters)g(will)i(b)q(e)150 1025
y(input,)15 b(and)f(th)o(us)f(unc)o(hanged)i(on)e(exit)h(from)f(the)h
(routine.)20 b(With)14 b(these)g(p)q(oin)o(ts)g(in)h(mind,)f(the)g
(calling)150 1082 y(sequences)h(and)g(parameter)e(declarations)i(for)f
(these)h(routines)f(are)g(giv)o(en)h(in)g(the)g(follo)o(wing)g
(sections.)150 1172 y Fk(vGSUM2D\()j(ICONTXT,)h(SCOPE,)g(TOP)-5
b(,)19 b(M,)g(N,)f(A)p 1302 1179 43 2 v(,)g(LD)n(A,)h(RDEST,)f(CDEST)h
(\))150 1230 y(vGAMX2D\()g(ICONTXT,)g(SCOPE,)g(TOP)-5
b(,)19 b(M,)f(N,)g(A)p 1312 1237 V(,)h(LD)n(A,)f(RA)p
1540 1237 85 2 v(,)h(CA)p 1659 1237 83 2 v(,)473 1288
y(R)n(CFLA)n(G,)53 b(RDEST,)18 b(CDEST)h(\))150 1347
y(vGAMN2D\()f(ICONTXT,)h(SCOPE,)g(TOP)-5 b(,)19 b(M,)f(N,)h(A)p
1314 1354 43 2 v(,)f(LD)n(A,)h(RA)p 1542 1354 85 2 v(,)f(CA)p
1660 1354 83 2 v 1 w(,)473 1405 y(R)n(CFLA)n(G,)53 b(RDEST,)18
b(CDEST)h(\))150 1493 y Fq(P)o(arameters:)210 1606 y(ICONTXT)110
b Fs(\(input\))16 b(INTEGER)570 1663 y(The)f(BLA)o(CS)h(con)o(text)f
(handle.)210 1752 y Fq(SCOPE)184 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
1808 y(Scop)q(e)24 b(of)e(pro)q(cesses)i(to)e(participate)h(in)h(op)q
(eration.)43 b(Limited)25 b(to)d Fl('ROW')p Fs(,)570
1864 y Fl('COLUMN')p Fs(,)14 b(or)g Fl('ALL')p Fs(.)g(See)i(Section)g
(4.2)f(for)f(additional)j(details.)210 1953 y Fq(TOP)249
b Fs(\(input\))16 b(CHARA)o(CTER*1)570 2010 y(Net)o(w)o(ork)f(top)q
(ology)h(to)g(b)q(e)g(em)o(ulated)h(during)g(comm)o(unication.)24
b(T)l(op)q(ologies)570 2066 y(presen)o(tly)16 b(supp)q(orted)g(are)e
(discussed)j(in)f(Section)g(8.4.)210 2155 y Fq(M)310
b Fs(\(input\))16 b(INTEGER)570 2211 y(The)f(n)o(um)o(b)q(er)h(of)f
(matrix)g(ro)o(ws)f(to)g(b)q(e)i(com)o(bined.)210 2300
y Fq(N)319 b Fs(\(input\))16 b(INTEGER)570 2357 y(The)f(n)o(um)o(b)q
(er)h(of)f(matrix)g(columns)h(to)e(b)q(e)i(com)o(bined.)210
2445 y Fq(A)320 b Fs(\(input/output\))15 b(TYPE)g(arra)o(y)f(\(LD)o(A,)
g(N\))570 2502 y(A)h(p)q(oin)o(ter)h(to)e(the)i(b)q(eginning)h(of)e
(the)g(\(sub\)arra)o(y)f(to)g(b)q(e)i(com)o(bined.)210
2591 y Fq(LD)o(A)250 b Fs(\(input\))16 b(INTEGER)570
2647 y(The)e(leading)i(dimension)g(of)d(the)h(matrix)g(A,)g(i.e.,)g
(the)g(distance)h(b)q(et)o(w)o(een)f(t)o(w)o(o)570 2704
y(successiv)o(e)i(elemen)o(ts)g(in)g(a)f(matrix)g(ro)o(w.)1027
2828 y(18)p eop
%%Page: 19 23
19 22 bop 210 199 a Fq(RA)281 b Fs(\(output\))14 b(INTEGER)i(arra)o(y)e
(\(R)o(CFLA)o(G,)g(N\))570 256 y(If)22 b(R)o(CFLA)o(G)g(=)g(-1,)i(this)
e(arra)o(y)f(will)j(not)e(b)q(e)h(referenced,)h(and)e(need)h(not)570
312 y(exist.)35 b(Otherwise)21 b(it)f(is)h(an)f(in)o(teger)g(arra)o(y)f
(\(of)g(size)i(at)e(least)h(R)o(CFLA)o(G)g(x)570 369
y(N\))f(indicating)j(the)d(ro)o(w)g(index)i(of)e(the)h(pro)q(cess)g
(that)f(pro)o(vided)h(the)g(maxi-)570 425 y(m)o(um/minim)o(um.)32
b(If)19 b(the)h(calling)g(pro)q(cess)g(is)f(not)g(selected)h(to)f
(receiv)o(e)h(the)570 482 y(result,)15 b(this)h(arra)o(y)e(will)j(con)o
(tain)e(in)o(termediate)h(\(useless\))g(results.)210
578 y Fq(CA)282 b Fs(\(output\))14 b(INTEGER)i(arra)o(y)e(\(R)o(CFLA)o
(G,)g(N\))570 635 y(If)i(R)o(CFLA)o(G)g(=)g(-1,)g(this)h(arra)o(y)e
(will)i(not)f(b)q(e)h(referenced,)g(and)f(need)h(not)f(ex-)570
691 y(ist.)29 b(Otherwise)19 b(it)g(is)f(an)g(in)o(teger)h(arra)o(y)e
(\(of)g(size)i(at)f(least)g(R)o(CFLA)o(G)g(x)g(N\))570
747 y(indicating)j(the)f(column)h(index)f(of)f(the)h(pro)q(cess)g(that)
f(pro)o(vided)h(the)g(maxi-)570 804 y(m)o(um/minim)o(um.)32
b(If)19 b(the)h(calling)g(pro)q(cess)g(is)f(not)g(selected)h(to)f
(receiv)o(e)h(the)570 860 y(result,)15 b(this)h(arra)o(y)e(will)j(con)o
(tain)e(in)o(termediate)h(\(useless\))g(results.)210
957 y Fq(R)o(CFLA)o(G)140 b Fs(\(input\))16 b(INTEGER)570
1013 y(If)f(R)o(CFLA)o(G)f(=)h(-1,)f(then)i(the)e(arra)o(ys)g(RA)h(and)
g(CA)f(are)h(not)f(referenced)i(and)570 1070 y(need)h(not)f(exist.)23
b(Otherwise,)16 b(R)o(CFLA)o(G)g(indicates)i(the)e(leading)h(dimension)
570 1126 y(of)e(these)g(arra)o(ys,)f(and)h(so)g(m)o(ust)f(b)q(e)i
Fn(\025)g Fs(M.)210 1223 y Fq(RDEST)182 b Fs(\(input\))16
b(INTEGER)570 1279 y(The)g(pro)q(cess)g(ro)o(w)f(co)q(ordinate)i(of)e
(the)h(pro)q(cess)g(who)g(should)h(receiv)o(e)g(the)f(re-)570
1336 y(sult.)32 b(If)19 b(RDEST)g(or)g(CDEST)f(=)i(-1,)f(all)h(pro)q
(cesses)g(within)g(the)f(indicated)570 1392 y(scop)q(e)d(receiv)o(e)g
(the)f(answ)o(er.)210 1489 y Fq(CDEST)183 b Fs(\(input\))16
b(INTEGER)570 1545 y(The)g(pro)q(cess)g(column)g(co)q(ordinate)g(of)g
(the)f(pro)q(cess)h(who)g(should)g(receiv)o(e)h(the)570
1602 y(result.)23 b(If)17 b(RDEST)f(or)g(CDEST)f(=)i(-1,)f(all)h(pro)q
(cesses)g(within)g(the)f(indicated)570 1658 y(scop)q(e)g(receiv)o(e)g
(the)f(answ)o(er.)150 1897 y Fk(8.3)56 b(Example)150
1985 y Fs(An)20 b(example)h(should)g(demonstrate)e(ho)o(w)h(these)g
(routines)h(are)e(used.)35 b(Assume)21 b(w)o(e)e(ha)o(v)o(e)h(a)g(2)f
(x)h(4)150 2041 y(pro)q(cess)15 b(grid)h(\(as)e(sho)o(wn)h(in)h(Figure)
f(1\).)k(Pro)q(cess)c Fn(f)p Fs(1,3)p Fn(g)f Fs(needs)i(the)f(maxim)o
(um)g(of)g(the)g(matrix)g Fl(B)g Fs(\(of)150 2098 y(size)h(4)f(x)g(4\))
g(o)o(v)o(er)f(all)i(pro)q(cesses.)k(All)d(pro)q(cesses)f(w)o(ould)f
(mak)o(e)g(the)g(follo)o(wing)h(call:)222 2154 y Fl(CALL)23
b(DGMAX2D\(ICONTXT,)f('ALL',)h(')g(',)h(4,)g(4,)f(B,)h(4,)f(RA,)h(CA,)f
(4,)h(1,)f(3\))150 2210 y Fs(Up)q(on)15 b(completion,)g(pro)q(cess)f
Fn(f)p Fs(1,3)p Fn(g)f Fs(w)o(ould)h(ha)o(v)o(e)g(three)g(matrices)g
(that)f(con)o(tain)i(information)f(on)g(the)150 2267
y(maximize)19 b(function.)29 b(The)19 b(matrix)e Fl(B)h
Fs(is)h(still)g(of)f(size)h(4)f(x)g(4.)28 b(Elemen)o(t)18
b(\(1,2\))f(of)g Fl(B)h Fs(w)o(ould)h(con)o(tain)150
2323 y(the)d(elemen)o(t)h(with)f(the)g(largest)g(absolute)g(v)m(alue)h
(found)g(on)e(an)o(y)h(pro)q(cess)g(at)g(matrix)f(lo)q(cation)i
(\(1,2\).)150 2380 y(RA\(1,2\))10 b(w)o(ould)h(indicate)i(what)e(pro)q
(cess)g(ro)o(w)f(that)g(maxim)o(um)h(w)o(as)g(found)g(on,)h(while)g
(CA\(1,2\))e(w)o(ould)150 2436 y(tell)16 b(whic)o(h)g(pro)q(cess)g
(column)g(it)f(w)o(as)g(found)g(on.)221 2493 y(As)k(another)f(example,)
j(assume)e(that)f(pro)q(cess)i(ro)o(w)e(1)h(requires)g(the)h(minim)o
(um)g(of)e(the)i(double)150 2550 y(precision)e(scalar)f
Fl(DMIN)p Fs(,)f(and)g(there)h(is)g(no)g(need)h(to)e(kno)o(w)g(what)g
(pro)q(cess)h(p)q(ossessed)g(the)g(min.)25 b(The)150
2606 y(co)q(de)16 b(w)o(ould)f(then)h(b)q(e:)269 2704
y Fl(IF)24 b(\(MYPROW)f(.EQ.)g(1\))h(THEN)1027 2828 y
Fs(19)p eop
%%Page: 20 24
20 23 bop 341 199 a Fl(CALL)23 b(DGAMN2D\(ICONTXT,)f('ROW',)h(')h(',)f
(1,)h(1,)f(DMIN,)h(1,)f(I,)h(I,)f(-1,)h(-1,)f(-1\))269
256 y(END)h(IF)150 398 y Fk(8.4)56 b(T)-5 b(op)r(ologies)150
490 y Fs(In)18 b(broadcasts,)f(the)h(BLA)o(CS)g(pro)o(vide)g(b)q(oth)g
(pip)q(elini)q(ng)j(and)c(non-pip)q(eli)q(nin)q(g)j(top)q(ologies.)28
b(A)o(t)17 b(the)150 547 y(momen)o(t,)12 b(the)g(BLA)o(CS)g(pro)o(vide)
g(pip)q(elin)q(ing)j(com)o(bine)e(top)q(ologies)f(only)h(in)g(the)f
(MPI)g(v)o(ersion.)19 b(There-)150 603 y(fore,)12 b(the)g(user)h(is)f
(encouraged)h(to)f(use)g(the)g(default)h(top)q(ology)f(\()p
Fl(TOP)23 b(=)h(')g(')p Fs(\))12 b(when)g(calling)i(a)e(com)o(bine)150
660 y(op)q(eration.)19 b(The)14 b(default)f Fl(TOP)g
Fs(option)g(will)i(attempt)d(to)h(use)g(the)g(top)q(ology)g(whic)o(h)h
(will)h(minimize)g(the)150 716 y(cost)g(of)f(one)i(call)g(to)f(a)g(com)
o(bine)h(op)q(eration.)221 776 y(App)q(endix)h(F)e(pro)o(vides)g
(detailed)i(descriptions)g(of)e(presen)o(tly)g(supp)q(orted)h(top)q
(ologies.)150 940 y Fr(9)69 b(Supp)r(ort)24 b(Routines)150
1048 y Fs(There)19 b(are)f(a)g(n)o(um)o(b)q(er)h(of)g(routines)g(whic)o
(h)g(do)g(not)f(deal)h(directly)h(with)f(comm)o(unication)g(that)f(are)
150 1105 y(nonetheless)h(required)f(for)f(programming)f(in)j(a)e
(parallel)i(en)o(vironmen)o(t.)26 b(The)18 b(BLA)o(CS)g(lab)q(el)h
(these)150 1161 y(routines)f(as)g Fm(supp)n(ort)g Fs(routines.)28
b(W)l(e)18 b(break)g(these)g(supp)q(ort)g(routines)g(in)o(to)g(rough)g
(categories,)g(and)150 1217 y(these)d(are)g(discussed)i(in)f(turn)f(b)q
(elo)o(w.)150 1359 y Fk(9.1)56 b(Initialization)150 1452
y Fs(These)15 b(routines)g(deal)g(with)g(grid/con)o(text)g(creation,)f
(and)h(pro)q(cessing)g(b)q(efore)g(the)g(grid/con)o(text)f(has)150
1509 y(b)q(een)i(de\014ned.)150 1649 y Fq(9.1.1)52 b(BLA)o(CS)p
487 1649 16 2 v 18 w(PINF)o(O)150 1742 y Fk(BLA)n(CS)p
338 1742 17 2 v 21 w(PINF)n(O\()19 b(MYPNUM)p 575 1749
274 2 v 1 w(,)f(NPR)n(OCS)p 883 1749 237 2 v 20 w(\))210
1858 y Fq(MYPNUM)103 b Fs(\(output\))14 b(INTEGER)570
1915 y(An)21 b(in)o(teger)g(b)q(et)o(w)o(een)g(0)f(and)h(\(NPR)o(OCS)f
(-)h(1\))f(whic)o(h)i(uniquely)g(iden)o(ti\014es)570
1971 y(eac)o(h)15 b(pro)q(cess.)210 2079 y Fq(NPR)o(OCS)139
b Fs(\(output\))14 b(INTEGER)570 2135 y(The)h(n)o(um)o(b)q(er)h(of)f
(pro)q(cesses)g(a)o(v)m(ailable)i(for)e(BLA)o(CS)g(use.)150
2252 y(This)e(routine)g(is)g(used)g(when)g(some)g(initial)h(system)e
(information)h(is)g(required)h(b)q(efore)f(the)f(BLA)o(CS)h(are)150
2308 y(set)g(up.)20 b(On)14 b(all)g(platforms)e(except)i(PVM,)f(NPR)o
(OCS)h(is)f(the)h(actual)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)g(a)o(v)
m(ailable)150 2365 y(for)e(use)h(\(i.e.)18 b(NPR)o(O)o(WS)12
b(*)f(NPCOLS)i Fn(\024)e Fs(NPR)o(OCS\).)h(In)g(PVM,)f(the)g(virtual)h
(mac)o(hine)h(ma)o(y)d(not)h(ha)o(v)o(e)150 2421 y(b)q(een)j(set)e(up)h
(b)q(efore)g(this)g(call,)h(and)f(therefore)g(no)f(parallel)i(mac)o
(hine)g(exists.)19 b(In)13 b(this)h(case,)e(NPR)o(OCS)150
2478 y(will)18 b(b)q(e)g(returned)f(as)f(less)h(than)g(one.)24
b(If)17 b(a)g(pro)q(cess)g(has)f(b)q(een)i(spa)o(wned)f(via)g(the)g(k)o
(eyb)q(oard,)f(it)h(will)150 2534 y(receiv)o(e)h(MYPNUM)f(of)f(0,)h
(and)g(all)h(other)f(pro)q(cesses)g(will)i(get)e(MYPNUM)f(of)h(-1.)25
b(This)18 b(allo)o(ws)f(the)150 2591 y(user)g(to)g(distinguish)i(b)q
(et)o(w)o(een)e(pro)q(cesses,)h(so)f(that)f(only)i(one)f(reads)g(in)h
(data,)f(etc.)25 b(Only)18 b(after)f(the)150 2647 y(virtual)e(mac)o
(hine)h(has)e(b)q(een)i(set)e(up)h(\(via)f(a)g(call)i(to)e(BLA)o(CS)p
1217 2647 14 2 v 17 w(SETUP)g(or)g(SETPVMTIDS\))g(will)j(this)150
2704 y(routine)f(return)f(the)g(correct)g(v)m(alues)h(for)f(MYPNUM)g
(and)g(NPR)o(OCS.)1027 2828 y(20)p eop
%%Page: 21 25
21 24 bop 150 199 a Fq(9.1.2)52 b(BLA)o(CS)p 487 199
16 2 v 18 w(SETUP)150 286 y Fk(BLA)n(CS)p 338 286 17
2 v 21 w(SETUP\()19 b(MYPNUM)p 585 293 274 2 v 1 w(,)f(NPR)n(OCS)p
893 293 237 2 v 20 w(\))210 399 y Fq(MYPNUM)103 b Fs(\(output\))14
b(INTEGER)570 456 y(An)21 b(in)o(teger)g(b)q(et)o(w)o(een)g(0)f(and)h
(\(NPR)o(OCS)f(-)h(1\))f(whic)o(h)i(uniquely)g(iden)o(ti\014es)570
512 y(eac)o(h)15 b(pro)q(cess.)210 608 y Fq(NPR)o(OCS)139
b Fs(\(input/output\))15 b(INTEGER)570 664 y(On)c(the)f(pro)q(cess)g
(spa)o(wned)h(from)e(the)h(k)o(eyb)q(oard)g(\(rather)g(than)g(from)f(p)
o(vmspa)o(wn\),)570 721 y(this)15 b(parameter)f(is)h(input,)h(and)f
(indicates)h(the)f(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)i(to)e(cre-)570
777 y(ate)h(when)g(building)j(the)d(virtual)g(mac)o(hine.)21
b(F)l(or)14 b(all)i(other)f(pro)q(cesses,)g(it)g(will)570
834 y(b)q(e)h(output.)150 947 y(This)j(routine)g(only)g(accomplishes)g
(meaningful)h(w)o(ork)e(in)h(the)f(PVM)g(BLA)o(CS.)g(On)h(all)h(other)e
(plat-)150 1004 y(forms,)13 b(it)h(is)g(functionally)h(equiv)m(alen)o
(t)g(to)e(BLA)o(CS)p 1045 1004 14 2 v 17 w(PINF)o(O.)h(The)g(BLA)o(CS)g
(assume)f(a)g(static)h(system:)150 1060 y(y)o(ou)e(start)f(with)i(a)f
(giv)o(en)h(n)o(um)o(b)q(er)f(of)g(pro)q(cesses,)h(and)g(that)e(is)i
(all)g(y)o(ou)f(will)i(ev)o(er)e(ha)o(v)o(e.)19 b(PVM)12
b(supplies)150 1117 y(a)g(dynamic)h(system,)g(allo)o(wing)g(pro)q
(cesses)g(to)e(b)q(e)j(added)f(to)e(the)i(system)f(on)g(the)h(\015y)l
(.)19 b(BLA)o(CS)p 1782 1117 V 17 w(SETUP)150 1173 y(is)13
b(used)h(to)e(actually)h(allo)q(cate)h(the)f(virtual)g(mac)o(hine)h
(and)f(spa)o(wn)f(pro)q(cesses.)20 b(It)13 b(reads)f(in)i(a)f(\014le)g
(called)150 1229 y Fl(blacs)p 273 1229 15 2 v 16 w(setup.dat)p
Fs(,)j(whose)h(\014rst)f(line)j(m)o(ust)d(b)q(e)i(the)f(name)g(of)f(y)o
(our)h(executable.)26 b(The)17 b(second)g(line)150 1286
y(is)h(optional,)f(but)g(if)h(it)f(exists,)h(it)f(should)h(b)q(e)g(a)f
(PVM)f(spa)o(wn)h(\015ag.)25 b(Legal)18 b(v)m(alues)g(at)e(this)i(time)
f(are)150 1342 y(0)g(\(PvmT)l(askDefault\),)f(4)h(\(PvmT)l(askDebug\),)
g(8)g(\(PvmT)l(askT)l(race\),)f(and)h(12)g(\(PvmT)l(askDebug)g(+)150
1399 y(PvmT)l(askT)l(race\).)i(The)14 b(primary)h(reason)f(for)g(this)h
(line)h(is)f(to)f(allo)o(w)g(the)h(user)g(to)e(easily)j(turn)e(on)h
(and)150 1455 y(o\013)j(PVM)h(debugging.)32 b(Additional)21
b(lines,)g(if)f(an)o(y)l(,)f(sp)q(ecify)i(what)d(mac)o(hines)i(should)g
(b)q(e)g(added)g(to)150 1512 y(the)13 b(curren)o(t)g(con\014guration)h
(b)q(efore)f(spa)o(wning)h(NPR)o(OCS-1)f(pro)q(cesses)h(to)f(the)g(mac)
o(hines)h(in)g(a)f(round)150 1568 y(robin)20 b(fashion.)34
b(NPR)o(OCS)20 b(is)g(input)g(on)g(the)g(pro)q(cess)f(whic)o(h)i(has)e
(no)h(PVM)f(paren)o(t)g(\(i.e.)33 b(MYP-)150 1625 y(NUM=0\),)16
b(and)g(b)q(oth)g(parameters)g(are)f(output)h(for)g(all)h(pro)q
(cesses.)23 b(Therefore,)16 b(on)g(PVM)g(systems,)150
1681 y(the)g(call)h(to)e(BLA)o(CS)p 523 1681 14 2 v 17
w(PINF)o(O)h(informs)g(y)o(ou)g(that)f(the)h(virtual)h(mac)o(hine)f
(has)g(not)g(b)q(een)h(set)f(up,)g(and)150 1738 y(a)h(call)i(to)d(BLA)o
(CS)p 487 1738 V 17 w(SETUP)i(then)f(sets)g(up)h(the)g(mac)o(hine)g
(and)f(returns)h(the)f(real)h(v)m(alues)g(for)f(MYP-)150
1794 y(NUM)d(and)g(NPR)o(OCS.)g(Note)f(that)g(if)i(the)e(\014le)i
Fl(blacs)p 1095 1794 15 2 v 17 w(setup.dat)d Fs(do)q(es)j(not)e(exist,)
h(the)g(BLA)o(CS)g(will)150 1850 y(prompt)19 b(the)h(user)g(for)f(the)h
(executable)h(name,)g(and)f(pro)q(cesses)g(will)i(b)q(e)e(spa)o(wned)g
(to)f(the)h(curren)o(t)150 1907 y(PVM)15 b(con\014guration.)150
2030 y Fq(9.1.3)52 b(BLA)o(CS)p 487 2030 16 2 v 18 w(GRIDINIT)150
2116 y Fk(BLA)n(CS)p 338 2116 17 2 v 21 w(GRIDINIT\()18
b(ICONTXT,)g(ORDER,)g(NPR)n(O)n(W,)h(NPCOL)g(\))210 2230
y Fq(ICONTXT)110 b Fs(\(input/output\))15 b(INTEGER)570
2286 y(On)h(input,)g(an)f(in)o(teger)h(handle)g(indicating)i(the)d
(system)g(con)o(text)g(to)f(b)q(e)i(used)570 2343 y(in)d(creating)g
(the)g(BLA)o(CS)g(con)o(text.)18 b(The)13 b(user)g(ma)o(y)e(obtain)i(a)
f(default)h(system)570 2399 y(con)o(text)j(via)i(a)e(call)i(to)e(BLA)o
(CS)p 1145 2399 14 2 v 17 w(GET.)g(On)i(output,)e(the)h(in)o(teger)g
(handle)i(to)570 2456 y(the)c(created)g(BLA)o(CS)h(con)o(text.)210
2551 y Fq(ORDER)169 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
2608 y(Indicates)g(ho)o(w)f(to)g(map)g(pro)q(cesses)g(to)g(BLA)o(CS)h
(grid.)k(Choices)c(are:)551 2704 y(`R')60 b(:)20 b(Use)15
b(ro)o(w-ma)s(jor)e(natural)i(ordering.)1027 2828 y(21)p
eop
%%Page: 22 26
22 25 bop 552 199 a Fs(`C')59 b(:)20 b(Use)15 b(column-ma)s(jor)g
(natural)g(ordering.)494 277 y(ELSE)61 b(:)20 b(Use)15
b(ro)o(w-ma)s(jor)e(natural)i(ordering.)210 381 y Fq(NPR)o(O)o(W)153
b Fs(\(input\))16 b(INTEGER)570 437 y(Indicates)g(ho)o(w)f(man)o(y)g
(pro)q(cess)g(ro)o(ws)g(the)g(pro)q(cess)g(grid)h(should)g(con)o(tain.)
210 541 y Fq(NPCOL)175 b Fs(\(input\))16 b(INTEGER)570
597 y(Indicates)g(ho)o(w)f(man)o(y)g(pro)q(cess)g(columns)h(the)g(pro)q
(cess)f(grid)h(should)g(con)o(tain.)150 713 y(All)c(BLA)o(CS)e(co)q
(des)h(m)o(ust)f(call)h(this)g(routine,)g(or)f(its)g(companion)h
(routine)g(BLA)o(CS)p 1574 713 14 2 v 17 w(GRIDMAP)l(.)f(These)150
769 y(routines)15 b(tak)o(e)e(the)h(a)o(v)m(ailable)i(pro)q(cesses,)e
(and)h(assign,)f(or)f(map,)h(them)g(in)o(to)g(a)g(BLA)o(CS)h(pro)q
(cess)f(grid.)150 826 y(In)i(other)f(w)o(ords,)f(they)h(establish)h(ho)
o(w)f(the)g(BLA)o(CS)g(co)q(ordinate)h(system)f(will)i(map)d(in)o(to)i
(the)f(nativ)o(e)150 882 y(mac)o(hine's)h(pro)q(cess)f(n)o(um)o(b)q
(ering)h(system.)k(Eac)o(h)15 b(BLA)o(CS)h(grid)f(is)h(con)o(tained)g
(in)g(a)f(con)o(text)f(\(its)h(o)o(wn)150 938 y(message)i(passing)g
(univ)o(erse\),)h(so)f(that)f(it)h(do)q(es)h(not)e(in)o(terfere)i(with)
f(distributed)i(op)q(erations)e(whic)o(h)150 995 y(o)q(ccur)h(within)h
(other)e(grids/con)o(texts.)26 b(These)18 b(grid)g(creation)g(routines)
g(ma)o(y)e(b)q(e)j(called)g(rep)q(eatedly)150 1051 y(in)d(order)f(to)g
(de\014ne)h(additional)g(con)o(texts/grids.)221 1110
y(The)i(creation)g(of)f(a)h(grid)g(requires)h(input)g(from)e(all)i(pro)
q(cesses)f(whic)o(h)h(are)e(de\014ned)j(to)d(b)q(e)h(in)h(it.)150
1167 y(It)14 b(is)g(therefore)f(a)h(globally-blo)q(c)o(king)i
(\(sometimes)d(called)j(syncronous\))d(op)q(eration)h(\(see)f(app)q
(endix)j(B)150 1223 y(for)h(details)h(on)g(blo)q(c)o(king\))g(whic)o(h)
h(means)e(that)g(pro)q(cesses)h(b)q(elonging)h(to)d(more)i(than)f(one)g
(grid)h(will)150 1280 y(ha)o(v)o(e)d(to)f(agree)h(on)g(whic)o(h)h(grid)
g(formation)e(will)j(b)q(e)f(serviced)g(\014rst.)221
1339 y(These)d(grid)h(creation)f(routines)h(set)f(up)g(v)m(arious)h(in)
o(ternals)g(for)e(the)i(BLA)o(CS,)f(and)g(so)g(one)g(of)g(them)150
1395 y(m)o(ust)i(b)q(e)g(called)i(b)q(efore)f(an)o(y)f(calls)h(are)f
(made)g(to)g(the)g(non-initialization)j(BLA)o(CS.)221
1454 y(Note)13 b(that)g(these)h(routines)g(map)g(already-existing)h
(pro)q(cesses)g(to)e(a)g(grid:)20 b(the)14 b(pro)q(cesses)g(are)f(not)
150 1510 y(created)19 b(dynamically)l(.)34 b(On)20 b(most)f(parallel)h
(mac)o(hines,)h(the)e(pro)q(cesses)h(will)h(b)q(e)f(actual)f(pro)q
(cessors)150 1567 y(\(hardw)o(are\),)f(and)i(they)f(are)g(\\created")g
(when)g(the)h(user)f(runs)g(his)h(executable.)33 b(When)20
b(using)g(the)150 1623 y(PVM)15 b(BLA)o(CS,)g(if)h(the)g(virtual)g(mac)
o(hine)g(has)f(not)g(b)q(een)h(set)f(up)h(y)o(et,)f(the)g(routine)h
(BLA)o(CS)p 1782 1623 V 17 w(SETUP)150 1680 y(should)21
b(b)q(e)g(used)g(to)e(create)h(the)h(virtual)g(mac)o(hine.)35
b(If)21 b(the)f(PVM)g(user)g(wishes)h(to)f(use)g(a)g(virtual)150
1736 y(mac)o(hine)h(already)g(set)f(up)h(using)g(explicit)i(PVM)d
(calls,)i(the)f(routine)g(SETPVMTIDS)f(should)h(b)q(e)150
1793 y(used)16 b(instead)g(of)e(BLA)o(CS)p 616 1793 V
17 w(SETUP)l(.)221 1852 y(This)24 b(routine)g(creates)f(a)h(simple)h
(NPR)o(O)o(W)e(x)h(NPCOL)g(pro)q(cess)g(grid.)45 b(This)25
b(pro)q(cess)f(grid)150 1908 y(will)f(use)f(the)g(\014rst)f(NPR)o(O)o
(W)g(*)g(NPCOL)h(pro)q(cesses,)h(and)f(assign)g(them)f(to)g(the)g(grid)
h(in)h(a)e(ro)o(w-)150 1964 y(or)h(column-ma)s(jor)f(natural)h
(ordering.)41 b(If)23 b(these)f(pro)q(cess-to-grid)g(mappings)h(are)f
(unacceptable,)150 2021 y(BLA)o(CS)p 304 2021 V 17 w(GRIDINIT's)14
b(more)g(complex)h(companion)f(routine)h(BLA)o(CS)p 1410
2021 V 17 w(GRIDMAP)e(m)o(ust)h(b)q(e)g(called)150 2077
y(instead.)150 2211 y Fq(9.1.4)52 b(BLA)o(CS)p 487 2211
16 2 v 18 w(GRIDMAP)150 2304 y Fk(BLA)n(CS)p 338 2304
17 2 v 21 w(GRIDMAP\()19 b(ICONTXT,)g(USERMAP)-5 b(,)19
b(LDUMAP)-5 b(,)19 b(NPR)n(O)n(W,)680 2362 y(NPCOL)g(\))210
2478 y Fq(ICONTXT)110 b Fs(\(input/output\))15 b(INTEGER)570
2534 y(On)h(input,)g(an)f(in)o(teger)h(handle)g(indicating)i(the)d
(system)g(con)o(text)g(to)f(b)q(e)i(used)570 2591 y(in)d(creating)g
(the)g(BLA)o(CS)g(con)o(text.)18 b(The)13 b(user)g(ma)o(y)e(obtain)i(a)
f(default)h(system)570 2647 y(con)o(text)j(via)i(a)e(call)i(to)e(BLA)o
(CS)p 1145 2647 14 2 v 17 w(GET.)g(On)i(output,)e(the)h(in)o(teger)g
(handle)i(to)570 2704 y(the)c(created)g(BLA)o(CS)h(con)o(text.)1027
2828 y(22)p eop
%%Page: 23 27
23 26 bop 210 199 a Fq(USERMAP)92 b Fs(\(input\))16 b(INTEGER)f(arra)o
(y)l(,)f(dimension)j(\(LDUMAP)l(,)d(NPCOL\))570 256 y(Input)i(arra)o(y)
e(indicating)j(the)e(pro)q(cess-to-grid)h(mapping.)210
368 y Fq(LDUMAP)123 b Fs(\(input\))16 b(INTEGER)570 424
y(The)f(leading)i(dimension)g(of)e(the)g(2D)g(arra)o(y)e(USERMAP)l(.)
210 536 y Fq(NPR)o(O)o(W)153 b Fs(\(input\))16 b(INTEGER)570
593 y(Indicates)g(ho)o(w)f(man)o(y)g(pro)q(cess)g(ro)o(ws)g(the)g(pro)q
(cess)g(grid)h(should)g(con)o(tain.)210 705 y Fq(NPCOL)175
b Fs(\(input\))16 b(INTEGER)570 761 y(Indicates)g(ho)o(w)f(man)o(y)g
(pro)q(cess)g(columns)h(the)g(pro)q(cess)f(grid)h(should)g(con)o(tain.)
150 879 y(All)c(BLA)o(CS)e(co)q(des)h(m)o(ust)f(call)h(this)g(routine,)
g(or)f(its)g(companion)h(routine)g(BLA)o(CS)p 1574 879
14 2 v 17 w(GRIDMAP)l(.)f(These)150 935 y(routines)15
b(tak)o(e)e(the)h(a)o(v)m(ailable)i(pro)q(cesses,)e(and)h(assign,)f(or)
f(map,)h(them)g(in)o(to)g(a)g(BLA)o(CS)h(pro)q(cess)f(grid.)150
992 y(In)i(other)f(w)o(ords,)f(they)h(establish)h(ho)o(w)f(the)g(BLA)o
(CS)g(co)q(ordinate)h(system)f(will)i(map)d(in)o(to)i(the)f(nativ)o(e)
150 1048 y(mac)o(hine's)h(pro)q(cess)f(n)o(um)o(b)q(ering)h(system.)k
(Eac)o(h)15 b(BLA)o(CS)h(grid)f(is)h(con)o(tained)g(in)g(a)f(con)o
(text)f(\(its)h(o)o(wn)150 1105 y(message)i(passing)g(univ)o(erse\),)h
(so)f(that)f(it)h(do)q(es)h(not)e(in)o(terfere)i(with)f(distributed)i
(op)q(erations)e(whic)o(h)150 1161 y(o)q(ccur)h(within)h(other)e
(grids/con)o(texts.)26 b(These)18 b(grid)g(creation)g(routines)g(ma)o
(y)e(b)q(e)j(called)g(rep)q(eatedly)150 1218 y(in)d(order)f(to)g
(de\014ne)h(additional)g(con)o(texts/grids.)221 1279
y(The)i(creation)g(of)f(a)h(grid)g(requires)h(input)g(from)e(all)i(pro)
q(cesses)f(whic)o(h)h(are)e(de\014ned)j(to)d(b)q(e)h(in)h(it.)150
1335 y(It)c(is)g(therefore)g(a)g(globally-blo)q(c)o(king)i(op)q
(eration)e(\(see)g(app)q(endix)i(B)e(for)f(details)i(on)f(blo)q(c)o
(king\))h(whic)o(h)150 1391 y(means)k(that)f(pro)q(cesses)h(b)q
(elonging)i(to)d(more)h(than)g(one)g(grid)g(will)i(ha)o(v)o(e)d(to)g
(agree)h(on)g(whic)o(h)h(grid)150 1448 y(formation)14
b(will)j(b)q(e)f(serviced)h(\014rst.)221 1509 y(These)c(grid)h
(creation)f(routines)h(set)f(up)g(v)m(arious)h(in)o(ternals)g(for)e
(the)i(BLA)o(CS,)f(and)g(so)g(one)g(of)g(them)150 1565
y(m)o(ust)i(b)q(e)g(called)i(b)q(efore)f(an)o(y)f(calls)h(are)f(made)g
(to)g(the)g(non-initialization)j(BLA)o(CS.)221 1626 y(Note)13
b(that)g(these)h(routines)g(map)g(already-existing)h(pro)q(cesses)g(to)
e(a)g(grid:)20 b(the)14 b(pro)q(cesses)g(are)f(not)150
1683 y(created)19 b(dynamically)l(.)34 b(On)20 b(most)f(parallel)h(mac)
o(hines,)h(the)e(pro)q(cesses)h(will)h(b)q(e)f(actual)f(pro)q(cessors)
150 1739 y(\(hardw)o(are\),)f(and)i(they)f(are)g(\\created")g(when)g
(the)h(user)f(runs)g(his)h(executable.)33 b(When)20 b(using)g(the)150
1796 y(PVM)15 b(BLA)o(CS,)g(if)h(the)g(virtual)g(mac)o(hine)g(has)f
(not)g(b)q(een)h(set)f(up)h(y)o(et,)f(the)g(routine)h(BLA)o(CS)p
1782 1796 V 17 w(SETUP)150 1852 y(should)21 b(b)q(e)g(used)g(to)e
(create)h(the)h(virtual)g(mac)o(hine.)35 b(If)21 b(the)f(PVM)g(user)g
(wishes)h(to)f(use)g(a)g(virtual)150 1909 y(mac)o(hine)h(already)g(set)
f(up)h(using)g(explicit)i(PVM)d(calls,)i(the)f(routine)g(SETPVMTIDS)f
(should)h(b)q(e)150 1965 y(used)16 b(instead)g(of)e(BLA)o(CS)p
616 1965 V 17 w(SETUP)l(.)221 2026 y(This)e(routine)g(allo)o(ws)g(the)g
(user)g(to)f(map)h(pro)q(cesses)g(to)f(the)h(pro)q(cess)g(grid)g(in)g
(an)g(arbitrary)f(manner.)150 2083 y(USERMAP\(i,j\))i(holds)h(the)g
(pro)q(cess)f(n)o(um)o(b)q(er)h(of)f(the)g(pro)q(cess)h(to)e(b)q(e)i
(placed)h(in)f Fn(f)p Fs(i,)g(j)p Fn(g)e Fs(of)h(the)h(pro)q(cess)150
2139 y(grid.)25 b(On)17 b(most)f(distributed)i(systems,)e(this)i(pro)q
(cess)f(n)o(um)o(b)q(er)g(will)h(simply)g(b)o(y)f(a)f(mac)o(hine)i
(de\014ned)150 2195 y(n)o(um)o(b)q(er)f(b)q(et)o(w)o(een)g(0)f
Fp(:)8 b(:)g(:)14 b Fs(NPR)o(OCS-1.)25 b(F)l(or)16 b(PVM)g(these)h(no)q
(de)g(n)o(um)o(b)q(ers)g(will)h(b)q(e)f(the)g(PVM)f(TIDS)150
2252 y(\(T)l(ask)g(IDs\).)23 b(BLA)o(CS)p 555 2252 V
17 w(GRIDMAP)16 b(is)h(not)f(for)g(the)h(inexp)q(erienced)i(user)e({)f
(BLA)o(CS)p 1669 2252 V 17 w(GRIDINIT)h(is)150 2308 y(m)o(uc)o(h)e
(simpler.)22 b(BLA)o(CS)p 603 2308 V 17 w(GRIDINIT)16
b(simply)h(p)q(erforms)e(a)g(GRIDMAP)g(where)h(the)f(\014rst)g(NPR)o(O)
o(W)150 2365 y(*)i(NPCOL)h(pro)q(cesses)g(are)g(mapp)q(ed)g(in)o(to)f
(the)h(curren)o(t)f(grid)h(in)h(a)e(ro)o(w-)g(or)g(column-ma)s(jor)g
(natural)150 2421 y(ordering.)j(BLA)o(CS)p 501 2421 V
17 w(GRIDMAP)15 b(allo)o(ws)g(the)g(exp)q(erienced)i(user)e(to)f(tak)o
(e)g(adv)m(an)o(tage)g(of)h(the)f(pro)q(ces-)150 2478
y(sors')h(actual)h(net)o(w)o(ork)f(\(i.e.)22 b(he)17
b(can)f(map)g(no)q(des)g(that)g(are)f(ph)o(ysically)j(connected)f(to)f
(b)q(e)g(neigh)o(b)q(ors)150 2534 y(in)j(the)f(BLA)o(CS)h(grid,)g
(etc.\).)29 b(BLA)o(CS)p 852 2534 V 16 w(GRIDMAP)19 b(also)f(op)q(ens)h
(the)f(w)o(a)o(y)f(for)h Fm(multigridding)p Fs(:)26 b(the)150
2591 y(user)14 b(can)h(separate)e(his)i(no)q(des)g(in)o(to)f(arbitrary)
f(grids,)i(join)f(them)g(together)g(at)f(some)h(later)g(date,)g(and)150
2647 y(then)19 b(re-split)h(them)f(in)o(to)g(new)g(grids.)31
b(BLA)o(CS)p 1022 2647 V 17 w(GRIDMAP)19 b(also)g(pro)o(vides)g(the)g
(abilit)o(y)h(to)e(mak)o(e)150 2704 y(arbitrary)h(grids)h(or)f
(subgrids)h(\(e.g.,)g(a)f(\\nearest)g(neigh)o(b)q(or")h(grid\),)g(whic)
o(h)h(can)f(greatly)f(facilitate)1027 2828 y(23)p eop
%%Page: 24 28
24 27 bop 150 199 a Fs(op)q(erations)19 b(among)f(groups)g(of)g(pro)q
(cesses)h(whic)o(h)g(do)g(not)f(fall)i(on)e(a)g(ro)o(w)g(or)g(column)i
(of)e(the)h(main)150 256 y(pro)q(cess)c(grid.)150 405
y Fk(9.2)56 b(Destruction)150 500 y Fs(These)16 b(routines)f(destro)o
(y)g(grids,)g(free)g(resources,)g(etc.)150 648 y Fq(9.2.1)52
b(BLA)o(CS)p 487 648 16 2 v 18 w(FREEBUFF)150 743 y Fk(BLA)n(CS)p
338 743 17 2 v 21 w(FREEBUFF\()18 b(ICONTXT,)h(W)-6 b(AIT)19
b(\))210 861 y Fq(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570
917 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h(con)o(text.)
210 1030 y Fq(W)-6 b(AIT)216 b Fs(\(input\))16 b(INTEGER)570
1087 y(Whether)f(to)g(w)o(ait)g(on)g(non-blo)q(c)o(king)i(op)q
(erations:)570 1143 y(IF)e(\(W)-5 b(AIT)15 b(.EQ.)g(0\))f(THEN)594
1200 y(Do)g(not)h(w)o(ait)g(on)g(op)q(erations,)g(free)g(only)h(un)o
(used)g(bu\013ers.)570 1256 y(ELSE)594 1312 y(If)f(necessary)l(,)h(w)o
(ait)e(in)i(order)f(to)g(free)g(all)h(bu\013ers.)570
1369 y(END)f(IF)150 1487 y(The)e(BLA)o(CS)h(ha)o(v)o(e)f(at)f(least)h
(one)g(in)o(ternal)h(bu\013er)f(that)g(is)g(used)h(for)f(pac)o(king)g
(messages)g(\(the)g(n)o(um)o(b)q(er)150 1543 y(of)k(in)o(ternal)i
(bu\013ers)e(v)m(aries,)i(dep)q(ending)g(on)f(whic)o(h)g(BLA)o(CS)g(y)o
(ou)g(are)f(using\).)27 b(On)19 b(systems)e(where)150
1600 y(memory)d(is)h(tigh)o(t,)g(k)o(eeping)g(this)h(bu\013er\(s\))e
(around)g(ma)o(y)g(b)q(ecome)i(exp)q(ensiv)o(e.)21 b(Calling)16
b(this)f(routine)150 1656 y(will)g(release)f(the)g(BLA)o(CS)g
(bu\013er\(s\).)k(Ho)o(w)o(ev)o(er,)13 b(the)h(next)f(call)i(to)e(a)g
(comm)o(unication)h(routine)g(whic)o(h)150 1712 y(requires)i(pac)o
(king)g(will)g(cause)g(the)f(bu\013er)g(to)g(b)q(e)h(reallo)q(cated.)
221 1774 y(The)d(parameter)f(W)-5 b(AIT)13 b(determines)h(whether)f
(the)g(BLA)o(CS)h(should)g(w)o(ait)e(for)h(an)o(y)f(non-blo)q(c)o(king)
150 1830 y(op)q(erations)19 b(to)g(complete)h(or)e(not.)31
b(If)20 b(W)-5 b(AIT)19 b(=)g(0,)h(the)f(BLA)o(CS)h(will)g(free)g(an)o
(y)e(bu\013ers)h(that)g(can)150 1887 y(b)q(e)k(freed)f(without)h(w)o
(aiting.)41 b(If)22 b(W)-5 b(AIT)22 b(is)h(not)f(0,)h(the)f(BLA)o(CS)h
(will)h(free)e(all)h(in)o(ternal)g(bu\013ers,)150 1943
y(p)q(ossibly)13 b(causing)f(the)f(call)i(to)e(blo)q(c)o(k)h(while)h
(the)e(BLA)o(CS)h(w)o(ait)f(for)f(in)o(ternal)j(non-blo)q(c)o(king)g
(op)q(erations)150 2000 y(complete.)150 2147 y Fq(9.2.2)52
b(BLA)o(CS)p 487 2147 16 2 v 18 w(GRIDEXIT)150 2242 y
Fk(BLA)n(CS)p 338 2242 17 2 v 21 w(GRIDEXIT\()18 b(ICONTXT)h(\))210
2360 y Fq(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570
2416 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h(con)o(text)e
(to)h(b)q(e)h(freed.)150 2534 y(Con)o(texts)f(consume)h(resources,)g
(and)g(therefore)g(the)g(user)g(should)h(release)f(them)g(when)g(they)g
(are)g(no)150 2591 y(longer)21 b(needed.)36 b(BLA)o(CS)p
630 2591 14 2 v 17 w(GRIDEXIT)21 b(frees)f(a)g(con)o(text.)34
b(After)20 b(the)h(freeing)f(of)g(a)g(con)o(text,)h(the)150
2647 y(con)o(text)14 b(no)g(longer)g(exists,)g(and)g(its)h(handle)g(ma)
o(y)e(b)q(e)i(re-issued)h(b)o(y)e(the)g(BLA)o(CS)g(if)h(a)f(new)g(con)o
(text)f(is)150 2704 y(de\014ned.)1027 2828 y(24)p eop
%%Page: 25 29
25 28 bop 150 199 a Fq(9.2.3)52 b(BLA)o(CS)p 487 199
16 2 v 18 w(ABOR)l(T)150 291 y Fk(BLA)n(CS)p 338 291
17 2 v 21 w(ABOR)-5 b(T\()19 b(ICONTXT,)g(ERR)n(ORNUM)f(\))210
407 y Fq(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570
464 y(In)o(teger)j(handle)i(indicating)h(the)d(BLA)o(CS)h(con)o(text)f
(whic)o(h)h(is)g(ab)q(orting)g(the)570 520 y(run.)210
627 y Fq(ERR)o(ORNUM)61 b Fs(\(input\))15 b(INTEGER)570
683 y(User)g(de\014ned)i(in)o(teger)e(error)g(n)o(um)o(b)q(er.)150
799 y(When)c(a)f(catastrophic)g(error)f(o)q(ccurs,)i(the)g(user)f(ma)o
(y)g(need)h(to)e(ab)q(ort)h(all)h(pro)q(cesses.)19 b(BLA)o(CS)p
1772 799 14 2 v 17 w(ABOR)l(T)150 856 y(exists)14 b(for)g(this)g
(reason.)19 b(Note)14 b(that)f(b)q(oth)h(parameters)g(are)f(input,)i
(but)f(that)g(BLA)o(CS)p 1677 856 V 17 w(ABOR)l(T)h(uses)150
912 y(them)d(only)i(in)f(prin)o(ting)g(out)f(the)h(error)f(message.)18
b(The)13 b(con)o(text)f(handle)i(passed)e(in)i(ma)o(y)e(b)q(e)h(an)o
(ything)150 969 y(\(i.e.,)18 b(it)h(need)g(not)e(b)q(e)i(a)f(v)m(alid)i
(con)o(text)d(handle\).)30 b(This)18 b(routine)h(kills)h(all)f(BLA)o
(CS)f(pro)q(cesses,)h(not)150 1025 y(just)c(those)g(con\014ned)h(to)f
(a)g(particular)g(con)o(text.)150 1163 y Fq(9.2.4)52
b(BLA)o(CS)p 487 1163 16 2 v 18 w(EXIT)150 1255 y Fk(BLA)n(CS)p
338 1255 17 2 v 21 w(EXIT\()18 b(CONTINUE)h(\))210 1371
y Fq(CONTINUE)71 b Fs(\(input\))16 b(INTEGER)570 1428
y(If)i(CONTINUE)g(is)g(non-zero,)g(it)g(is)g(assumed)g(that)f(the)g
(user)h(will)h(con)o(tin)o(ue)570 1484 y(using)12 b(the)g(mac)o(hine)h
(after)e(the)g(BLA)o(CS)h(are)g(done.)19 b(Otherwise,)13
b(it)f(is)g(assumed)570 1540 y(that)i(no)i(message)e(passing)i(will)h
(b)q(e)f(done)f(after)g(the)g(BLA)o(CS)p 1662 1540 14
2 v 17 w(EXIT)g(call.)150 1657 y(This)g(routine)g(should)h(b)q(e)f
(called)h(when)f(a)f(pro)q(cess)h(has)f(\014nished)i(all)g(use)f(of)f
(the)g(BLA)o(CS.)h(It)g(frees)f(all)150 1713 y(BLA)o(CS)e(con)o(texts)f
(and)g(releases)h(all)h(memory)d(the)i(BLA)o(CS)g(ha)o(v)o(e)f(allo)q
(cated.)20 b(CONTINUE)12 b(indicates)150 1769 y(whether)19
b(the)f(user)h(will)h(b)q(e)f(using)h(the)e(underlying)j(comm)o
(unication)e(platform)f(after)g(the)g(BLA)o(CS)150 1826
y(are)f(\014nished.)27 b(This)17 b(information)g(is)h(most)e(imp)q
(ortan)o(t)h(for)f(the)h(PVM)g(BLA)o(CS.)g(If)g(CONTINUE)g(is)150
1882 y(set)f(to)g(0,)h(then)f Fl(pvm)p 511 1882 15 2
v 17 w(exit)g Fs(will)j(b)q(e)e(called;)h(otherwise,)f(it)g(will)h
(not.)24 b(If)17 b(the)f(user)h(sets)f(CONTINUE)150 1939
y(not)i(equal)g(to)g(0,)f(he)i(is)f(indicating)i(that)d(he)i(will)g(b)q
(e)g(calling)h(explicit)g(PVM)d(send/recvs)i(after)e(the)150
1995 y(BLA)o(CS)c(are)g(done,)g(so)f(that)g(the)h(pro)q(cess)g(cannot)g
(tell)g(the)g(virtual)h(mac)o(hine)f(that)f(it)h(is)g(done.)20
b(It)13 b(then)150 2052 y(b)q(ecomes)j(the)g(user's)f(resp)q(onsibilit)
o(y)j(to)d(mak)o(e)g(sure)g(his)i(co)q(de)f(calls)g Fl(pvm)p
1441 2052 V 17 w(exit)p Fs(.)k(PVM)15 b(users)h(should)150
2108 y(either)g(call)g(BLA)o(CS)p 516 2108 14 2 v 17
w(EXIT)g(or)e(explicitly)k(call)f Fl(pvm)p 1071 2108
15 2 v 16 w(exit)e Fs(to)f(a)o(v)o(oid)h(PVM)g(problems.)150
2248 y Fk(9.3)56 b(Informational)17 b(and)i(Miscellaneous)150
2340 y Fs(These)i(routines)g(return)f(information)h(in)o(v)o(olving)h
(the)e(pro)q(cess)h(grid.)36 b(Also)21 b(included)i(here)e(is)g(the)150
2396 y(barrier)15 b(routine.)150 2534 y Fq(9.3.1)52 b(BLA)o(CS)p
487 2534 16 2 v 18 w(GRIDINF)o(O)150 2628 y Fk(BLA)n(CS)p
338 2628 17 2 v 21 w(GRIDINF)n(O\()19 b(ICONTXT,)f(NPR)n(O)n(W)p
988 2635 221 2 v 1 w(,)g(NPCOL)p 1243 2635 199 2 v 19
w(MYPR)n(O)n(W)p 1460 2635 273 2 v 2 w(,)687 2686 y(MYPCOL)p
687 2693 250 2 v 19 w(\))1027 2828 y Fs(25)p eop
%%Page: 26 30
26 29 bop 210 199 a Fq(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 256 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h
(con)o(text)e(to)h(b)q(e)h(queried.)210 352 y Fq(NPR)o(O)o(W)153
b Fs(\(output\))14 b(INTEGER)570 408 y(On)i(output,)e(the)i(n)o(um)o(b)
q(er)f(of)g(pro)q(cess)g(ro)o(ws)g(in)h(ICONTXT's)f(pro)q(cess)g(grid.)
210 504 y Fq(NPCOL)175 b Fs(\(output\))14 b(INTEGER)570
560 y(On)21 b(output,)h(the)f(n)o(um)o(b)q(er)g(of)f(pro)q(cess)h
(columns)g(in)h(ICONTXT's)f(pro)q(cess)570 617 y(grid.)210
713 y Fq(MYPR)o(O)o(W)104 b Fs(\(output\))14 b(INTEGER)570
769 y(On)i(output,)e(the)i(calling)h(pro)q(cess's)e(ro)o(w)f(co)q
(ordinate)h(in)h(the)g(pro)q(cess)f(grid.)210 865 y Fq(MYPCOL)126
b Fs(\(output\))14 b(INTEGER)570 922 y(On)e(output,)g(the)g(calling)i
(pro)q(cess's)e(column)h(co)q(ordinate)f(in)h(the)f(pro)q(cess)g(grid.)
150 1035 y(Returns)h(information)f(ab)q(out)g(the)h(pro)q(cess)f(grid)h
(con)o(tained)g(in)g(the)f(con)o(text)g(whose)g(handle)h(is)g(ICON-)150
1091 y(TXT.)i(If)g(the)g(con)o(text)g(handle)h(is)g(in)o(v)m(alid,)h
(all)f(quan)o(tities)g(are)f(returned)h(as)e(-1.)150
1214 y Fq(9.3.2)52 b(BLA)o(CS)p 487 1214 16 2 v 18 w(PNUM)150
1301 y Fk(INTEGER)18 b(FUNCTION)h(BLA)n(CS)p 947 1301
17 2 v 21 w(PNUM)p 759 1308 385 2 v(\()g(ICONTXT,)f(PR)n(O)n(W,)i(PCOL)
f(\))210 1415 y Fq(ICONTXT)110 b Fs(\(input\))16 b(In)o(teger)f(handle)
h(indicating)h(the)f(BLA)o(CS)f(con)o(text)g(to)f(b)q(e)i(queried.)210
1567 y Fq(PR)o(O)o(W)194 b Fs(\(input\))16 b(INTEGER)570
1623 y(The)f(ro)o(w)e(co)q(ordinate)i(of)f(the)h(pro)q(cess)g(whose)f
(system)g(pro)q(cess)h(n)o(um)o(b)q(er)g(is)g(to)570
1680 y(b)q(e)h(determined.)210 1776 y Fq(PCOL)216 b Fs(\(input\))16
b(INTEGER)570 1832 y(The)i(column)g(co)q(ordinate)g(of)g(the)f(pro)q
(cess)h(whose)g(system)f(pro)q(cess)h(n)o(um)o(b)q(er)570
1889 y(is)e(to)e(b)q(e)i(determined.)150 2002 y(This)k(function)g
(returns)g(the)f(system)g(pro)q(cess)h(n)o(um)o(b)q(er)g(\(i.e.,)f(a)h
(task)e(ID)i(for)f(PVM)g(users\))g(of)g(the)150 2059
y(pro)q(cess)c(at)g Fn(f)p Fs(PR)o(O)o(W,)f(PCOL)p Fn(g)i
Fs(in)g(the)f(pro)q(cess)h(grid.)150 2182 y Fq(9.3.3)52
b(BLA)o(CS)p 487 2182 16 2 v 18 w(PCOORD)150 2268 y Fk(BLA)n(CS)p
338 2268 17 2 v 21 w(PCOORD\()19 b(ICONTXT,)g(PNUM,)g(PR)n(O)n(W)p
1159 2275 177 2 v(,)g(PCOL)p 1370 2275 155 2 v 19 w(\))210
2382 y Fq(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570
2438 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h(con)o(text.)
210 2534 y Fq(PNUM)193 b Fs(\(input\))16 b(INTEGER)570
2591 y(The)g(pro)q(cess)g(n)o(um)o(b)q(er)f(whose)h(co)q(ordinates)g
(are)f(to)g(b)q(e)h(determined.)22 b(This)16 b(is)570
2647 y(the)g(pro)q(cess)h(n)o(um)o(b)q(er)g(of)f(the)g(underlying)j
(mac)o(hine)e(\(e.g.,)e(it)i(will)h(b)q(e)f(a)f(TID)570
2704 y(for)f(PVM\).)1027 2828 y(26)p eop
%%Page: 27 31
27 30 bop 210 199 a Fq(PR)o(O)o(W)194 b Fs(\(output\))14
b(INTEGER)570 256 y(On)i(output,)e(the)h(ro)o(w)f(co)q(ordinate)i(of)e
(pro)q(cess)i(PNUM)e(in)i(the)f(BLA)o(CS)h(grid.)210
363 y Fq(PCOL)216 b Fs(\(output\))14 b(INTEGER)570 420
y(On)19 b(output,)g(the)f(column)h(co)q(ordinate)g(of)f(pro)q(cess)h
(PNUM)f(in)h(the)f(BLA)o(CS)570 476 y(grid.)150 593 y(Giv)o(en)j(the)f
(system)g(pro)q(cess)g(n)o(um)o(b)q(er)h(\(i.e.,)g(a)f(task)f(ID)i(for)
e(PVM)h(users\),)h(returns)f(the)g(ro)o(w)g(and)150 649
y(column)c(co)q(ordinates)g(in)g(the)f(BLA)o(CS')g(pro)q(cess)h(grid.)
150 789 y Fq(9.3.4)52 b(BLA)o(CS)p 487 789 16 2 v 18
w(BARRIER)150 882 y Fk(BLA)n(CS)p 338 882 17 2 v 21 w(BARRIER\()18
b(ICONTXT,)g(SCOPE)i(\))210 998 y Fq(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 1055 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)
h(con)o(text.)210 1162 y Fq(SCOPE)184 b Fs(\(input\))16
b(CHARA)o(CTER*1)570 1219 y(Indicates)d(whether)f(a)f(pro)q(cess)h(ro)o
(w)e(\(SCOPE='R'\),)h(column)i(\('C'\),)c(or)i(en)o(tire)570
1275 y(grid)16 b(\('A'\))d(will)k(participate)f(in)g(barrier.)150
1392 y(This)f(routines)f(holds)h(up)f(execution)h(of)f(all)h(pro)q
(cesses)f(within)i(the)e(indicated)i(scop)q(e)e(un)o(til)h(they)f(ha)o
(v)o(e)150 1448 y(all)i(called)h(the)e(routine.)150 1590
y Fk(9.4)56 b(General)17 b(purp)r(ose)150 1682 y Fs(The)g(BLA)o(CS)g
(ha)o(v)o(e)f(t)o(w)o(o)g(general)h(purp)q(ose)g(routines.)25
b(They)17 b(are)f Fl(BLACS)p 1453 1682 15 2 v 17 w(SET)g
Fs(and)h Fl(BLACS)p 1768 1682 V 17 w(GET)p Fs(.)e(Be-)150
1739 y(cause)j(they)g(ma)o(y)f(b)q(e)i(called)g(b)q(efore)f(a)g(grid)g
(is)g(created,)g(they)g(are)g(often)f(lump)q(ed)j(in)e(with)g(the)g
(ini-)150 1795 y(tialization)h(routines.)28 b(These)18
b(routines)g(are)f(used)h(to)f(set)h(and)f(obtain)h(information)g(ab)q
(out)f(v)m(arious)150 1852 y(BLA)o(CS)h(in)o(ternals.)27
b(Some)17 b(of)g(these)h(in)o(ternals)g(con)o(trol)f(general)h(BLA)o
(CS)g(b)q(eha)o(vior,)g(and)f(are)g(th)o(us)150 1908
y(not)11 b(link)o(ed)j(to)d(a)g(particular)i(con)o(text.)18
b(These)12 b(in)o(ternals)g(will)i(ignore)e(the)g(parameter)f
Fl(ICONTXT)p Fs(,)f(whic)o(h)150 1965 y(exists)16 b(so)e(that)h(in)o
(ternals)h(whic)o(h)g(are)f(tied)h(to)e(a)h(particular)h(con)o(text)f
(ma)o(y)f(b)q(e)i(op)q(erated)f(on.)150 2105 y Fq(9.4.1)52
b(BLA)o(CS)p 487 2105 16 2 v 18 w(GET)150 2197 y Fk(BLA)n(CS)p
338 2197 17 2 v 21 w(GET\()18 b(ICONTXT,)h(WHA)-5 b(T,)19
b(V)-6 b(AL)p 1031 2204 113 2 v 18 w(\))210 2314 y Fq(ICONTXT)110
b Fs(\(input\))16 b(INTEGER)570 2370 y(On)j(WHA)l(Ts)f(that)f(are)h
(tied)h(to)f(a)f(particular)i(con)o(text,)f(this)h(is)g(the)f(in)o
(teger)570 2426 y(handle)j(indicating)g(the)e(BLA)o(CS)h(con)o(text)e
(to)h(query)l(.)32 b(Otherwise,)21 b(it)f(is)f(ig-)570
2483 y(nored.)210 2591 y Fq(WHA)l(T)193 b Fs(\(input\))16
b(INTEGER)570 2647 y(What)10 b(BLA)o(CS)h(in)o(ternal)h(information)f
(should)g(b)q(e)h(returned)f(in)g(V)-5 b(AL.)11 b(Presen)o(t)570
2704 y(options)k(are:)1027 2828 y(27)p eop
%%Page: 28 32
28 31 bop 570 155 1663 2 v 570 165 V 569 222 2 57 v 578
222 V 603 205 a Fs(WHA)l(T)p 771 222 V 50 w(Returned)16
b(in)g(V)-5 b(AL)p 2223 222 V 2232 222 V 570 223 1663
2 v 570 233 V 569 290 2 57 v 578 290 V 724 273 a(0)p
771 290 V 50 w(Handle)16 b(indicating)h(default)f(system)f(con)o(text)p
2223 290 V 2232 290 V 570 291 1663 2 v 569 348 2 57 v
578 348 V 724 331 a(1)p 771 348 V 50 w(The)g(BLA)o(CS)h(message)f(ID)g
(range)p 2223 348 V 2232 348 V 570 350 1663 2 v 569 406
2 57 v 578 406 V 724 389 a(2)p 771 406 V 50 w(The)g(BLA)o(CS)h(debug)g
(lev)o(el)p 2223 406 V 2232 406 V 570 408 1663 2 v 569
464 2 57 v 578 464 V 702 447 a(10)p 771 464 V 49 w(Handle)g(indicating)
h(the)f(system)e(con)o(text)h(used)h(to)e(de\014ne)j(the)p
2223 464 V 2232 464 V 569 521 V 578 521 V 771 521 V 797
504 a(BLA)o(CS)f(con)o(text)e(whose)h(handle)i(is)f(ICONTXT)p
2223 521 V 2232 521 V 570 522 1663 2 v 569 579 2 57 v
578 579 V 702 562 a(11)p 771 579 V 49 w(Num)o(b)q(er)g(of)e(rings)i(m)o
(ultiring)g(broadcast)f(top)q(ology)g(is)g(presen)o(tly)h(using)p
2223 579 V 2232 579 V 570 580 1663 2 v 569 637 2 57 v
578 637 V 702 620 a(12)p 771 637 V 49 w(Num)o(b)q(er)g(of)e(branc)o
(hes)i(general)g(tree)f(broadcast)f(top)q(ology)h(is)h(presen)o(tly)f
(using)p 2223 637 V 2232 637 V 570 638 1663 2 v 569 695
2 57 v 578 695 V 702 678 a(13)p 771 695 V 49 w(Num)o(b)q(er)h(of)e
(rings)i(m)o(ultiring)g(com)o(bine)g(top)q(ology)f(is)h(presen)o(tly)g
(using)p 2223 695 V 2232 695 V 570 697 1663 2 v 569 753
2 57 v 578 753 V 702 736 a(14)p 771 753 V 49 w(Num)o(b)q(er)g(of)e
(branc)o(hes)i(general)g(tree)f(com)o(bine)h(top)q(ology)e(is)i(presen)
o(tly)g(using)p 2223 753 V 2232 753 V 570 755 1663 2
v 569 811 2 57 v 578 811 V 702 794 a(15)p 771 811 V 49
w(If)f(top)q(ologies)h(are)f(b)q(eing)h(forced)g(to)e(b)q(e)i(rep)q
(eatable,)g(a)f(non-zero)g(is)p 2223 811 V 2232 811 V
569 868 V 578 868 V 771 868 V 797 851 a(returned.)20
b(If)c(rep)q(eatabilit)o(y)g(is)g(not)f(b)q(eing)h(enforced,)g(zero)f
(is)g(returned)p 2223 868 V 2232 868 V 569 924 V 578
924 V 771 924 V 797 907 a(\(see)g(app)q(endix)i(D)e(for)g(details)h(on)
f(rep)q(eatabilit)o(y\).)p 2223 924 V 2232 924 V 570
926 1663 2 v 569 982 2 57 v 578 982 V 702 965 a(16)p
771 982 V 49 w(If)g(top)q(ologies)h(are)f(b)q(eing)h(forced)g(to)e(b)q
(e)i(heterogeneous)f(coheren)o(t,)g(a)p 2223 982 V 2232
982 V 569 1039 V 578 1039 V 771 1039 V 797 1022 a(non-zero)g(is)h
(returned.)k(If)c(heterogeneous)f(coherence)h(is)g(not)f(b)q(eing)p
2223 1039 V 2232 1039 V 569 1095 V 578 1095 V 771 1095
V 797 1078 a(enforced,)g(zero)g(is)h(returned)g(\(see)f(app)q(endix)i
(D)e(for)f(details)i(on)g(coherence\).)p 2223 1095 V
2232 1095 V 570 1097 1663 2 v 210 1210 a Fq(V)-6 b(AL)255
b Fs(\(output\))14 b(INTEGER)i(ARRA)l(Y)g(of)f(v)m(ariable)i(dimension)
570 1267 y(The)10 b(v)m(alue)i(to)d(whic)o(h)i(the)g(BLA)o(CS)f(in)o
(ternal)h(is)g(presen)o(tly)g(set.)18 b(The)10 b(dimension)570
1323 y(of)15 b(V)-5 b(AL)15 b(is)h(\(2\))e(if)i(the)f(message)g(ID)g
(range)g(is)g(b)q(eing)i(returned.)j(F)l(or)14 b(all)i(other)570
1380 y(queries)g(it)g(is)f(\(1\).)150 1496 y(This)g(routine)f(returns)g
(the)h(v)m(alues)g(the)f(BLA)o(CS)h(are)e(using)i(for)f(in)o(ternal)h
(defaults.)20 b(Some)14 b(v)m(alues)h(are)150 1553 y(tied)e(to)e(a)h
(BLA)o(CS)h(con)o(text,)f(and)g(some)g(are)g(more)f(general.)20
b(The)12 b(most)f(common)h(use)g(is)h(in)g(retrieving)150
1609 y(a)18 b(default)h(system)f(con)o(text)f(for)h(input)h(in)o(to)f
(BLA)o(CS)p 1111 1609 14 2 v 17 w(GRIDINIT)h(or)f(BLA)o(CS)p
1576 1609 V 17 w(GRIDMAP)l(.)g(Some)150 1666 y(systems,)i(suc)o(h)f(as)
h(MPI,)f(supply)h(their)g(o)o(wn)f(v)o(ersion)h(of)f(con)o(text)g(\(in)
h(MPI,)f(this)h(corresp)q(onds)g(to)150 1722 y(a)e(comm)o(unicator\).)
28 b(F)l(or)17 b(those)h(users)g(who)g(mix)g(system)g(co)q(de)h(with)f
(BLA)o(CS)g(co)q(de,)h(w)o(e)f(therefore)150 1779 y(need)j(to)f(b)q(e)h
(able)g(to)e(form)h(a)g(BLA)o(CS)h(con)o(text)e(in)j(reference)f(to)e
(a)h(system)g(con)o(text.)35 b(Th)o(us,)21 b(the)150
1835 y(grid)15 b(creation)h(routines)f(tak)o(e)f(a)h(system)g(con)o
(text)f(as)h(input.)20 b(If)c(y)o(ou)e(wish)i(to)e(ha)o(v)o(e)h
(strictly)g(p)q(ortable)150 1892 y(co)q(de,)g(y)o(ou)f(ma)o(y)h(use)g
(BLA)o(CS)p 683 1892 V 16 w(GET)g(to)f(retriev)o(e)h(a)f(default)h
(system)g(con)o(text)f(whic)o(h)h(will)i(include)g(all)150
1948 y(a)o(v)m(ailable)g(pro)q(cesses.)221 2008 y(Also)f(not)g(tied)i
(to)d(a)i(particular)g(BLA)o(CS)g(con)o(text)f(are)g(the)g(message)g
(ID)h(range)f(and)h(the)f(debug)150 2065 y(lev)o(el)22
b(the)e(BLA)o(CS)h(w)o(ere)g(compiled)h(with.)36 b(F)l(or)20
b(these)h(three)f(v)m(alues)i(of)e(WHA)l(T,)g(the)h(parameter)150
2121 y(ICONTXT)16 b(is)f(not)g(referenced.)221 2182 y(The)c(other)g(c)o
(hoices)h(of)f(WHA)l(T)g(are)g(all)i(tied)f(to)e(a)h(particular)h(BLA)o
(CS)g(con)o(text,)f(so)g(the)h(parameter)150 2238 y(ICONTXT)k(m)o(ust)e
(b)q(e)i(a)f(v)m(alid)i(BLA)o(CS)e(con)o(text)g(handle.)150
2380 y Fq(9.4.2)52 b(BLA)o(CS)p 487 2380 16 2 v 18 w(SET)150
2474 y Fk(BLA)n(CS)p 338 2474 17 2 v 21 w(SET\()18 b(ICONTXT,)h(WHA)-5
b(T,)19 b(V)-6 b(AL)19 b(\))210 2591 y Fq(ICONTXT)110
b Fs(\(input\))16 b(INTEGER)570 2647 y(On)j(WHA)l(Ts)f(that)f(are)h
(tied)h(to)f(a)f(particular)i(con)o(text,)f(this)h(is)g(the)f(in)o
(teger)570 2704 y(handle)e(indicating)i(the)d(BLA)o(CS)h(con)o(text.)j
(Otherwise,)c(it)h(is)g(ignored.)1027 2828 y(28)p eop
%%Page: 29 33
29 32 bop 210 199 a Fq(WHA)l(T)193 b Fs(\(input\))16
b(INTEGER)570 256 y(What)f(BLA)o(CS)g(in)o(ternal\(s\))h(should)g(b)q
(e)g(set)f(to)f(V)-5 b(AL.)15 b(Presen)o(t)g(options)h(are:)p
570 273 1583 2 v 570 283 V 569 339 2 57 v 578 339 V 603
322 a(WHA)l(T)p 771 339 V 50 w(V)-5 b(AL)16 b(determines)p
2143 339 V 2152 339 V 570 341 1583 2 v 570 351 V 569
407 2 57 v 578 407 V 724 390 a(1)p 771 407 V 50 w(The)f(BLA)o(CS)h
(message)f(ID)g(range)p 2143 407 V 2152 407 V 570 409
1583 2 v 569 465 2 57 v 578 465 V 702 449 a(11)p 771
465 V 49 w(Num)o(b)q(er)h(of)e(rings)i(for)e(m)o(ultiring)j(broadcast)d
(top)q(ology)h(to)g(use)p 2143 465 V 2152 465 V 570 467
1583 2 v 569 524 2 57 v 578 524 V 702 507 a(12)p 771
524 V 49 w(Num)o(b)q(er)h(of)e(branc)o(hes)i(for)e(general)i(tree)f
(broadcast)g(top)q(ology)f(to)h(use)p 2143 524 V 2152
524 V 570 525 1583 2 v 569 582 2 57 v 578 582 V 702 565
a(13)p 771 582 V 49 w(Num)o(b)q(er)h(of)e(rings)i(for)e(m)o(ultiring)j
(com)o(bine)f(top)q(ology)f(to)f(use)p 2143 582 V 2152
582 V 570 583 1583 2 v 569 640 2 57 v 578 640 V 702 623
a(14)p 771 640 V 49 w(Num)o(b)q(er)i(of)e(branc)o(hes)i(for)e(general)i
(tree)f(com)o(bine)h(top)q(ology)f(to)g(use)p 2143 640
V 2152 640 V 570 642 1583 2 v 569 698 2 57 v 578 698
V 702 681 a(15)p 771 698 V 49 w(Whether)g(top)q(ologies)h(should)g(b)q
(e)g(forced)f(to)g(b)q(e)h(rep)q(eatable.)p 2143 698
V 2152 698 V 569 754 V 578 754 V 771 754 V 797 737 a(If)f(this)h(v)m
(alue)g(is)g(0)f(\(the)g(default\))g(top)q(ologies)h(are)f(not)g
(required)h(to)f(b)q(e)p 2143 754 V 2152 754 V 569 811
V 578 811 V 771 811 V 797 794 a(rep)q(eatable.)21 b(An)o(y)15
b(other)g(v)m(alue)h(requires)g(all)g(used)g(top)q(ologies)g(to)e(b)q
(e)p 2143 811 V 2152 811 V 569 867 V 578 867 V 771 867
V 797 850 a(rep)q(eatable.)21 b(\(see)15 b(app)q(endix)i(D)e(for)f
(details)i(on)g(rep)q(eatabilit)o(y\).)p 2143 867 V 2152
867 V 570 869 1583 2 v 569 925 2 57 v 578 925 V 702 908
a(16)p 771 925 V 49 w(Whether)f(top)q(ologies)h(should)g(b)q(e)g
(forced)f(to)g(b)q(e)h(heterogeneous)f(coheren)o(t)p
2143 925 V 2152 925 V 569 982 V 578 982 V 771 982 V 797
965 a(or)g(not.)k(If)d(this)f(v)m(alue)i(is)e(0)g(\(the)g(default\))h
(top)q(ologies)f(are)g(not)g(required)p 2143 982 V 2152
982 V 569 1038 V 578 1038 V 771 1038 V 797 1021 a(to)g(b)q(e)g
(heterogeneous)h(coheren)o(t.)k(An)o(y)15 b(other)g(v)m(alue)h
(requires)g(all)g(used)p 2143 1038 V 2152 1038 V 569
1095 V 578 1095 V 771 1095 V 797 1078 a(top)q(ologies)g(to)e(b)q(e)i
(heterogeneous)f(coheren)o(t)p 2143 1095 V 2152 1095
V 569 1151 V 578 1151 V 771 1151 V 797 1134 a(\(see)g(app)q(endix)i(D)e
(for)g(details)h(on)f(coherence\).)p 2143 1151 V 2152
1151 V 570 1153 1583 2 v 210 1248 a Fq(V)-6 b(AL)255
b Fs(\(input\))16 b(INTEGER)f(ARRA)l(Y)i(of)d(v)m(ariable)j(dimension)
570 1305 y(The)f(v)m(alue\(s\))h(to)e(set)h(in)o(ternals)g(to.)22
b(Its)16 b(sp)q(eci\014c)i(meaning)e(is)h(dep)q(enden)o(t)g(on)570
1361 y(WHA)l(T,)d(as)g(discussed)i(b)q(elo)o(w.)k(Note)14
b(that)g(for)g(WHA)l(T)g(=)h(1,)f(the)g(dimension)570
1418 y(of)h(V)-5 b(AL)15 b(is)h(\(2\).)j(Otherwise,)d(it)f(is)h(\(1\).)
150 1531 y(Sets)f(BLA)o(CS)h(in)o(ternal)g(defaults.)k(The)c(action)f
(tak)o(en)g(is)h(dep)q(enden)o(t)g(up)q(on)g(WHA)l(T,)f(as)g(follo)o
(ws:)206 1618 y(1.)22 b Fq(Setting)c(the)g(BLA)o(CS)f(message)g(ID)g
(range)264 1674 y Fs(If)h(the)f(user)h(wishes)h(to)e(mix)h(the)g(BLA)o
(CS)g(with)g(other)g(message-passing)g(pac)o(k)m(ages,)g(he)g(ma)o(y)
264 1731 y(restrict)f(the)g(BLA)o(CS)h(to)f(a)g(certain)h(message)f(ID)
g(range,)g(whic)o(h)i(he)e(ensures)h(is)g(not)f(used)h(b)o(y)264
1787 y(the)i(non-BLA)o(CS)i(routines.)36 b(The)21 b(message)f(ID)g
(range)h(m)o(ust)f(b)q(e)h(set)f(b)q(efore)h(the)g(\014rst)f(call)264
1844 y(to)e(BLA)o(CS)p 477 1844 14 2 v 17 w(GRIDMAP)h(or)g(BLA)o(CS)p
951 1844 V 17 w(GRIDINIT.)h(Subsequen)o(t)g(calls)h(will)f(ha)o(v)o(e)f
(no)h(e\013ect.)264 1900 y(Because)j(the)f(message)g(ID)g(range)g(is)h
(not)f(tied)h(to)e(a)h(particular)h(con)o(text,)g(the)g(parameter)264
1957 y(ICONTXT)15 b(is)h(ignored,)f(and)h(V)-5 b(AL)15
b(is)h(de\014ned)h(as:)324 2030 y Fq(V)-6 b(AL)255 b
Fs(\(input\))15 b(INTEGER)h(arra)o(y)e(of)g(dimension)j(\(2\))684
2087 y(V)-5 b(AL\(1\):)23 b(The)17 b(smallest)h(message)f(ID)g(\(also)g
(called)i(message)d(t)o(yp)q(e)h(or)684 2143 y(message)d(tag\))g(the)i
(BLA)o(CS)f(should)h(use.)684 2200 y(V)-5 b(AL\(2\):)29
b(The)20 b(largest)f(message)h(ID)g(\(also)f(called)j(message)d(t)o(yp)
q(e)h(or)684 2256 y(message)14 b(tag\))g(the)i(BLA)o(CS)f(should)h
(use.)183 2478 y(12.)22 b Fq(Set)17 b(n)o(um)o(b)q(er)g(of)g(rings)g
(for)g(TOP)h(=)f('M')f(\(m)o(ultiring)j(broadcast\))264
2534 y Fs(This)g(quan)o(tit)o(y)f(is)h(tied)g(to)f(a)g(con)o(text,)g
(th)o(us)g(ICONTXT)h(is)g(used,)g(and)g(V)-5 b(AL)19
b(is)g(de\014ned)h(as:)264 2591 y(V)-5 b(AL\(1\):)30
b(The)20 b(n)o(um)o(b)q(er)h(of)f(rings)h(for)f(m)o(ultiring)h(top)q
(ology)g(to)e(use.)37 b(V)l(alid)21 b(v)m(alues)h(are)e(are)264
2647 y(all)h(nonzero)g(n)o(um)o(b)q(ers,)h(where)g(negativ)o(e)e(n)o
(um)o(b)q(ers)i(corresp)q(ond)f(to)f(decreasing)i(rings,)g(and)264
2704 y(p)q(ositiv)o(e)f(n)o(um)o(b)q(ers)f(indicate)i(increasing)f
(rings.)35 b(Note)19 b(that)h(y)o(ou)g(cannot)f(ha)o(v)o(e)h(more)g
(rings)1027 2828 y(29)p eop
%%Page: 30 34
30 33 bop 264 199 a Fs(than)16 b(there)h(are)f(pro)q(cesses)h(in)g(the)
g(op)q(eration,)f(so)g(if)h Fp(N)1252 206 y Fo(p)1288
199 y Fs(is)g(the)g(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)h(in)g(the)264
256 y(op)q(eration,)j(then)g Fn(j)p Fs(V)-5 b(AL\(1\))o
Fn(j)19 b Fp(>)h Fs(N)876 263 y Ff(p)911 256 y Fn(\000)13
b Fs(1,)20 b(results)g(in)g(a)f(fully)i(connected)f(top)q(ology)f
(\(i.e.,)h(the)264 312 y(n)o(um)o(b)q(er)15 b(of)g(rings)g(will)i(b)q
(e)f(set)f(to)g Fp(N)904 319 y Fo(p)933 312 y Fn(\000)c
Fs(1\).)183 409 y(13.)22 b Fq(Set)17 b(n)o(um)o(b)q(er)g(of)g(branc)o
(hes)g(for)g(TOP)h(=)f('T')g(\(general)h(tree)f(broadcast\))264
465 y Fs(This)i(quan)o(tit)o(y)f(is)h(tied)g(to)f(a)g(con)o(text,)g(th)
o(us)g(ICONTXT)h(is)g(used,)g(and)g(V)-5 b(AL)19 b(is)g(de\014ned)h
(as:)264 522 y(V)-5 b(AL\(1\):)22 b(The)17 b(n)o(um)o(b)q(er)f(of)g
(branc)o(hes)h(for)f(general)h(tree)g(top)q(ology)f(to)g(use.)24
b(V)l(alid)18 b(v)m(alues)g(are:)264 578 y(V)-5 b(AL\(1\))12
b Fp(>)h Fs(0.)18 b(Note)12 b(that)f(y)o(ou)h(cannot)f(ha)o(v)o(e)h
(more)f(branc)o(hes)h(than)g(there)g(are)g(pro)q(cesses)g(in)h(the)264
635 y(op)q(eration,)e(so)g(if)g Fp(N)602 642 y Fo(p)633
635 y Fs(is)g(the)g(n)o(um)o(b)q(er)h(of)e(pro)q(cesses)i(in)f(the)h
(op)q(eration,)f(then)h Fn(j)p Fs(V)-5 b(AL\(1\))o Fn(j)12
b Fp(>)h Fs(N)1854 642 y Ff(p)1878 635 y Fn(\000)r Fs(1,)264
691 y(results)e(in)h(a)f(fully)h(connected)g(top)q(ology)f(\(i.e.,)g
(the)g(n)o(um)o(b)q(er)g(of)g(branc)o(hes)g(will)i(b)q(e)f(set)e(to)h
Fp(N)1839 698 y Fo(p)1860 691 y Fn(\000)r Fs(1\).)183
788 y(14.)22 b Fq(Set)17 b(n)o(um)o(b)q(er)g(of)g(rings)g(for)g(TOP)h
(=)f('M')f(\(m)o(ultiring)j(com)o(bine\))264 845 y Fs(This)g(quan)o
(tit)o(y)f(is)h(tied)g(to)f(a)g(con)o(text,)g(th)o(us)g(ICONTXT)h(is)g
(used,)g(and)g(V)-5 b(AL)19 b(is)g(de\014ned)h(as:)264
901 y(V)-5 b(AL\(1\):)27 b(The)19 b(n)o(um)o(b)q(er)h(of)e(rings)i(for)
e(m)o(ultiring)i(com)o(bine)g(top)q(ology)f(to)f(use.)32
b(V)l(alid)21 b(v)m(alues)264 958 y(are)14 b(are)h(all)g(nonzero)g(n)o
(um)o(b)q(ers,)g(where)g(negativ)o(e)g(n)o(um)o(b)q(ers)g(corresp)q
(ond)g(to)f(decreasing)i(rings,)264 1014 y(and)d(p)q(ositiv)o(e)h(n)o
(um)o(b)q(ers)f(indicate)i(increasing)f(rings.)19 b(Note)13
b(that)f(y)o(ou)h(cannot)g(ha)o(v)o(e)f(more)h(rings)264
1070 y(than)j(there)h(are)f(pro)q(cesses)h(in)g(the)g(op)q(eration,)f
(so)g(if)h Fp(N)1252 1077 y Fo(p)1288 1070 y Fs(is)g(the)g(n)o(um)o(b)q
(er)g(of)f(pro)q(cesses)h(in)g(the)264 1127 y(op)q(eration,)j(then)g
Fn(j)p Fs(V)-5 b(AL\(1\))o Fn(j)19 b Fp(>)h Fs(N)876
1134 y Ff(p)911 1127 y Fn(\000)13 b Fs(1,)20 b(results)g(in)g(a)f
(fully)i(connected)f(top)q(ology)f(\(i.e.,)h(the)264
1183 y(n)o(um)o(b)q(er)15 b(of)g(rings)g(will)i(b)q(e)f(set)f(to)g
Fp(N)904 1190 y Fo(p)933 1183 y Fn(\000)c Fs(1\).)183
1280 y(15.)22 b Fq(Set)17 b(n)o(um)o(b)q(er)g(of)g(branc)o(hes)g(for)g
(TOP)h(=)f('T')g(\(general)h(tree)f(gather\))264 1337
y Fs(This)i(quan)o(tit)o(y)f(is)h(tied)g(to)f(a)g(con)o(text,)g(th)o
(us)g(ICONTXT)h(is)g(used,)g(and)g(V)-5 b(AL)19 b(is)g(de\014ned)h(as:)
264 1393 y(V)-5 b(AL\(1\):)22 b(The)17 b(n)o(um)o(b)q(er)f(of)g(branc)o
(hes)h(for)f(general)h(tree)g(top)q(ology)f(to)g(use.)24
b(V)l(alid)18 b(v)m(alues)g(are:)264 1450 y(V)-5 b(AL\(1\))12
b Fp(>)h Fs(0.)18 b(Note)12 b(that)f(y)o(ou)h(cannot)f(ha)o(v)o(e)h
(more)f(branc)o(hes)h(than)g(there)g(are)g(pro)q(cesses)g(in)h(the)264
1506 y(op)q(eration,)e(so)g(if)g Fp(N)602 1513 y Fo(p)633
1506 y Fs(is)g(the)g(n)o(um)o(b)q(er)h(of)e(pro)q(cesses)i(in)f(the)h
(op)q(eration,)f(then)h Fn(j)p Fs(V)-5 b(AL\(1\))o Fn(j)12
b Fp(>)h Fs(N)1854 1513 y Ff(p)1878 1506 y Fn(\000)r
Fs(1,)264 1563 y(results)e(in)h(a)f(fully)h(connected)g(top)q(ology)f
(\(i.e.,)g(the)g(n)o(um)o(b)q(er)g(of)g(branc)o(hes)g(will)i(b)q(e)f
(set)e(to)h Fp(N)1839 1570 y Fo(p)1860 1563 y Fn(\000)r
Fs(1\).)183 1659 y(16.)22 b Fq(Set)17 b(whether)g(top)q(ologies)j(m)o
(ust)c(b)q(e)i(rep)q(eatable)h(or)e(not)264 1716 y Fs(If)f(this)g(v)m
(alue)h(is)f(set)g(to)f(0)g(\(the)h(default\),)g(top)q(ologies)g(are)f
(not)h(required)h(to)e(b)q(e)h(rep)q(eatable.)23 b(If)264
1772 y(it)13 b(is)h(set)f(to)g(an)o(y)g(v)m(alue)i(b)q(esides)g(0,)e
(all)h(top)q(ologies)g(will)h(b)q(e)f(forced)g(to)f(b)q(e)h(rep)q
(eatable)g(\(and)f(th)o(us)264 1829 y(p)q(ossibly)j(tak)o(e)f(a)g(p)q
(erformance)g(hit\).)20 b(See)c(App)q(endix)h(D)e(for)g(details)h(on)f
(rep)q(eatabilit)o(y)l(.)183 1926 y(17.)22 b Fq(Set)17
b(whether)g(top)q(ologies)j(m)o(ust)c(b)q(e)i(heterogeneous)f(coheren)o
(t)h(or)f(not)264 1982 y Fs(If)e(this)h(v)m(alue)g(is)g(set)f(to)g(0)g
(\(the)g(default\),)g(top)q(ologies)h(are)f(not)g(required)h(to)f(b)q
(e)h(heterogeneous)264 2039 y(coheren)o(t.)28 b(If)19
b(it)f(is)h(set)f(to)f(an)o(y)h(v)m(alue)i(b)q(esides)f(0,)g(all)g(top)
q(ologies)f(will)i(b)q(e)f(forced)f(to)g(b)q(e)h(het-)264
2095 y(erogeneous)14 b(coheren)o(t)h(\(and)g(th)o(us)g(p)q(ossibly)h
(tak)o(e)e(a)h(p)q(erformance)g(hit\).)20 b(See)15 b(App)q(endix)i(D)e
(for)264 2151 y(details)h(on)f(coherence.)150 2278 y
Fk(9.5)56 b(Uno\016cial)18 b(routines)150 2365 y Fs(These)j(routines)g
(are)f(not)g(part)g(of)g(the)h(BLA)o(CS)g(standard,)g(and)g(th)o(us)f
(not)g(guaran)o(teed)g(to)g(b)q(e)h(in)150 2421 y(ev)o(ery)e(BLA)o(CS)h
(implemen)o(tation.)34 b(Most)18 b(of)h(these)g(routines)h(ha)o(v)o(e)f
(v)m(alid)i(uses,)g(but)e(it)h(is)f(di\016cult)150 2478
y(to)e(defend)h(adding)g(them)f(to)g(a)g(message-passing)g(standard.)26
b(SETPVMTIDS)17 b(is)h(system)e(sp)q(eci\014c,)150 2534
y(and)k(so)g(ob)o(viously)g(w)o(ould)g(not)g(\014t)g(in)o(to)g(the)g
(standard.)33 b(The)20 b(timing)h(routines)f(are)g(quite)g(useful,)150
2591 y(as)c(they)h(allo)o(w)g(for)f(system-indep)q(enden)o(t)i(timing,)
g(but)e(timing)i(do)q(es)e(not)h(ha)o(v)o(e)f(a)g(great)g(deal)h(to)f
(do)150 2647 y(with)f(message)f(passing.)20 b(Finally)l(,)c(as)e(a)g
(service)h(to)f(the)h(user,)f(w)o(e)g(allo)o(w)h(him)g(to)f(access)g
(the)h(BLA)o(CS')150 2704 y(message)21 b(ID)h(computation)g(routines.)
40 b(F)l(or)22 b(the)g(user)g(mixing)g(primitiv)o(e)i(\(i.e.)40
b(system-sp)q(eci\014c\))1027 2828 y(30)p eop
%%Page: 31 35
31 34 bop 150 199 a Fs(message)15 b(passing)h(with)g(the)f(BLA)o(CS,)h
(these)f(routines)h(ma)o(y)f(b)q(e)h(con)o(v)o(enien)o(t.)21
b(None)16 b(of)f(these)h(things)150 256 y(\014t)j(in)o(to)g(a)g
(message)f(passing)i(standard,)f(and)g(so)g(they)g(are)g(pro)o(vided)h
(b)o(y)f(the)g(presen)o(t)g(BLA)o(CS)g(as)150 312 y(uno\016cial)d
(service)h(routines.)150 430 y Fq(9.5.1)52 b(SETPVMTIDS)150
515 y Fk(SETPVMTIDS\()19 b(NT)-5 b(ASKS,)19 b(TIDS)f(\))210
628 y Fq(NT)l(ASKS)148 b Fs(\(input\))16 b(INTEGER)570
685 y(The)f(n)o(um)o(b)q(er)h(of)f(PVM)g(tasks)f(the)h(user)h(has)f
(spa)o(wned.)210 773 y Fq(TIDS)235 b Fs(\(input\))16
b(INTEGER)f(arra)o(y)f(of)h(dimension)i(\(NT)l(ASKS\))570
829 y(This)e(arra)o(y)d(con)o(tains)i(the)g(list)h(of)e(the)h(NT)l
(ASKS)h(PVM)f(task)f(IDS)i(whic)o(h)f(will)570 886 y(participate)i(in)g
(the)f(BLA)o(CS.)150 999 y(SETPVMTIDS,)21 b(as)h(its)g(name)f(implies,)
k(is)e(a)e(PVM)g(sp)q(eci\014c)j(routine.)40 b(SETPVMTIDS)22
b(is)g(the)150 1055 y(adv)m(anced)14 b(PVM)f(user's)g(BLA)o(CS)p
745 1055 14 2 v 17 w(SETUP)l(.)g(BLA)o(CS)p 1089 1055
V 17 w(SETUP)g(ma)o(y)g(b)q(e)h(to)q(o)f(restrictiv)o(e)g(for)g
(someone)150 1112 y(who)20 b(is)h(using)g(PVM)f(outside)h(the)f(BLA)o
(CS.)h(F)l(or)e(example,)j(they)f(ma)o(y)e(w)o(an)o(t)g(to)h(start)f
(the)i(main)150 1168 y(pro)q(cess)16 b(\(pro)q(cess)f(0,0\))g(via)h(a)f
(call)i(to)e Fl(pvm)p 905 1168 15 2 v 16 w(spawn)p Fs(,)g(rather)g
(than)g(starting)g(it)h(from)f(the)h(k)o(eyb)q(oard)f(as)150
1225 y(BLA)o(CS)p 304 1225 14 2 v 17 w(SETUP)f(requires.)20
b(SETPVMTIDS)14 b(requires)h(t)o(w)o(o)e(parameters)g(from)g(the)h
(user.)20 b(The)14 b(\014rst)150 1281 y(is)i(the)g(total)f(n)o(um)o(b)q
(er)h(of)g(pro)q(cesses)g(\(or)f(tasks\))f(that)h(an)o(y)h(BLA)o(CS)g
(grid)g(will)h(use.)22 b(Remem)o(b)q(er)17 b(that)150
1337 y(the)h(BLA)o(CS)g(is)h(a)e(static)h(system:)25
b(if)18 b(y)o(ou)g(ha)o(v)o(e)f(P)h(pro)q(cesses)g(at)g(the)g(b)q
(eginning)i(of)d(its)h(execution,)150 1394 y(y)o(ou)f(m)o(ust)g(ha)o(v)
o(e)g(those)g(same)g(P)g(pro)q(cesses)h(when)g(the)f(BLA)o(CS)h
(\014nish)h(execution.)27 b(Therefore,)18 b(the)150 1450
y(user)d(m)o(ust)g(set)g(NT)l(ASKS)h(to)f(b)q(e)h(the)f(largest)g(n)o
(um)o(b)q(er)g(of)g(pro)q(cesses)h(he)f(will)i(ev)o(er)e(use.)221
1507 y(The)h(second)h(argumen)o(t)f(required)i(b)o(y)e(SETPVMTIDS)g(is)
h(a)g(list)g(of)f(TIDS)h(in)g(an)g(in)o(teger)f(arra)o(y)150
1563 y(of)e(at)f(least)i(length)f(NT)l(ASKS.)h(All)h(pro)q(cesses)e
(require)h(these)g(inputs.)20 b(This)15 b(means)f(that)g(in)h(order)f
(to)150 1620 y(use)k(SETPVMTIDS,)f(the)h(PVM)f(user)h(should)h(spa)o
(wn)e(all)i(of)e(his)i(pro)q(cesses,)f(k)o(eeping)h(their)f(TIDS)150
1676 y(in)j(an)e(in)o(teger)h(arra)o(y)l(,)g(then)g(send)h(that)e(arra)
o(y)f(to)h(all)i(participating)g(pro)q(cesses,)g(and)f(\014nally)h(ha)o
(v)o(e)150 1733 y(them)f(all)i(call)f(SETPVMTIDS.)f(A)o(t)g(this)h(p)q
(oin)o(t,)g(he)g(has)f(p)q(erformed)h(the)f(actions)h(inheren)o(t)g(in)
g(a)150 1789 y(BLA)o(CS)p 304 1789 V 17 w(SETUP)g(call,)j(and)e(he)g
(ma)o(y)f(then)g(pro)q(ceed)i(to)e(use)g(the)h(BLA)o(CS)g(as)f(usual)h
(\(can)f(mak)o(e)150 1846 y(calls)16 b(to)f(BLA)o(CS)p
461 1846 V 17 w(PINF)o(O,)g(and)g(then)h(to)e(BLA)o(CS)p
1049 1846 V 17 w(GRIDINIT)j(or)d(BLA)o(CS)p 1508 1846
V 17 w(GRIDMAP)l(,)h(and)h(then)150 1902 y(pro)q(ceed)g(with)g(the)f
(normal)g(BLA)o(CS)h(co)q(de\).)150 2019 y Fq(9.5.2)52
b(DCPUTIME)150 2105 y Fk(DOUBLE)18 b(PRECISION)h(FUNCTION)g(DCPUTIME)p
1069 2112 315 2 v(\()g(\))150 2162 y Fs(This)13 b(routine)g(returns)f
(time)g(\(in)h(seconds\))g(elapsed)g(since)g(an)g(arbitrary)e(starting)
h(p)q(oin)o(t.)19 b(W)l(e)13 b(roughly)150 2218 y(de\014ne)j(CPU)e
(time)h(to)e(b)q(e)i(the)g(time)g(the)f(pro)q(cessor)g(sp)q(ends)i
(actually)f(executing)h(user)e(co)q(de.)20 b(If)15 b(CPU)150
2275 y(time)h(is)f(not)g(a)o(v)m(ailable)i(on)e(a)g(giv)o(en)h(system,)
e Fl(-1.0)h Fs(is)g(returned.)150 2392 y Fq(9.5.3)52
b(D)o(W)-6 b(ALL)l(TIME)150 2478 y Fk(DOUBLE)18 b(PRECISION)h(FUNCTION)
g(D)n(W)-6 b(ALL)h(TIME)p 1069 2485 348 2 v(\()18 b(\))150
2534 y Fs(This)g(routine)g(returns)f(time)h(\(in)g(seconds\))f(elapsed)
i(since)g(an)e(arbitrary)g(starting)g(p)q(oin)o(t.)27
b(Here)17 b(w)o(e)150 2591 y(lo)q(osely)d(de\014ne)f(W)-5
b(ALL)14 b(time)e(to)g(b)q(e)i(the)e(time)h(y)o(ou)g(w)o(ould)g
(\014gure)f(if)i(y)o(ou)e(lo)q(ok)o(ed)h(at)f(the)h(clo)q(c)o(k)g(on)f
(y)o(our)150 2647 y(w)o(all,)17 b(b)q(egan)g(the)f(op)q(eration,)h(and)
g(then)g(subtracted)f(it)h(from)e(the)i(time)g(y)o(our)f(clo)q(c)o(k)h
(sho)o(w)o(ed)f(at)g(the)150 2704 y(end)g(of)f(the)g(op)q(eration.)20
b(If)15 b(W)-5 b(ALL)16 b(time)g(is)f(not)g(a)o(v)m(ailable)i(on)e(a)g
(giv)o(en)h(system,)e Fl(-1.0)h Fs(is)g(returned.)1027
2828 y(31)p eop
%%Page: 32 36
32 35 bop 150 199 a Fq(9.5.4)52 b(KSENDID)150 286 y Fk(INTEGER)18
b(FUNCTION)h(KSENDID)p 759 293 263 2 v -1 w(\()g(ICONTXT,)f(RDEST,)g
(CDEST)h(\))210 400 y Fq(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 456 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h
(con)o(text.)210 553 y Fq(RDEST)182 b Fs(\(input\))16
b(INTEGER)570 609 y(The)f(ro)o(w)g(destination)h(of)f(the)g(message)g
(that)f(needs)i(an)f(ID.)210 706 y Fq(CDEST)183 b Fs(\(input\))16
b(INTEGER)570 762 y(The)f(column)i(destination)f(of)e(the)i(message)e
(that)h(needs)h(an)f(ID.)150 876 y(Returns)h(a)f(BLA)o(CS)g(message)g
(ID)g(the)h(user)f(ma)o(y)f(safely)i(use)f(in)h(primitiv)o(e)h(send)f
(calls.)150 1000 y Fq(9.5.5)52 b(KRECVID)150 1087 y Fk(INTEGER)18
b(FUNCTION)h(KRECVID)p 759 1094 270 2 v(\()f(ICONTXT,)h(RSR)n(C,)g(CSR)
n(C)h(\))210 1200 y Fq(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 1257 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)
h(con)o(text.)210 1353 y Fq(RSR)o(C)216 b Fs(\(input\))16
b(INTEGER)570 1410 y(The)f(ro)o(w)g(source)g(of)g(the)g(message)g(that)
f(needs)i(an)f(ID.)210 1506 y Fq(CSR)o(C)217 b Fs(\(input\))16
b(INTEGER)570 1562 y(The)f(column)i(source)e(of)g(the)g(message)g(that)
f(needs)i(an)f(ID.)150 1676 y(Returns)h(a)f(BLA)o(CS)g(message)g(ID)g
(the)h(user)f(ma)o(y)f(safely)i(use)f(in)h(primitiv)o(e)h(receiv)o(e)f
(calls.)150 1800 y Fq(9.5.6)52 b(KBSID)150 1887 y Fk(INTEGER)18
b(FUNCTION)h(KBSID)p 759 1894 179 2 v -1 w(\()g(ICONTXT,)g(SCOPE)g(\))
210 2000 y Fq(ICONTXT)110 b Fs(\(input\))16 b(INTEGER)570
2057 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h(con)o(text.)
210 2153 y Fq(SCOPE)184 b Fs(\(input\))16 b(CHARA)o(CTER*1)570
2210 y(Indicates)d(whether)f(a)f(pro)q(cess)h(ro)o(w)e(\(SCOPE='R'\),)h
(column)i(\('C'\),)c(or)i(en)o(tire)570 2266 y(grid)16
b(\('A'\))d(will)k(participate)f(broadcast.)150 2380
y(Returns)h(a)f(BLA)o(CS)h(message)f(ID)g(the)h(user)f(ma)o(y)g(safely)
g(use)h(in)g(for)f(the)g(source)h(\(destination\))f(of)g(a)150
2436 y(primitiv)o(e)h(broadcast)d(\(com)o(bine\).)150
2560 y Fq(9.5.7)52 b(KBRID)150 2647 y Fk(INTEGER)18 b(FUNCTION)h(KBRID)
p 759 2654 190 2 v -1 w(\()g(ICONTXT,)f(SCOPE,)h(RSR)n(C,)g(CSR)n(C)h
(\))1027 2828 y Fs(32)p eop
%%Page: 33 37
33 36 bop 210 199 a Fq(ICONTXT)110 b Fs(\(input\))16
b(INTEGER)570 256 y(In)o(teger)f(handle)i(indicating)g(the)e(BLA)o(CS)h
(con)o(text.)210 350 y Fq(SCOPE)184 b Fs(\(input\))16
b(CHARA)o(CTER*1)570 406 y(Indicates)d(whether)f(a)f(pro)q(cess)h(ro)o
(w)e(\(SCOPE='R'\),)h(column)i(\('C'\),)c(or)i(en)o(tire)570
462 y(grid)16 b(\('A'\))d(will)k(participate)f(broadcast.)210
556 y Fq(RSR)o(C)216 b Fs(\(input\))16 b(INTEGER)570
613 y(The)f(ro)o(w)g(source)g(of)g(the)g(broadcast)f(message)h(that)g
(needs)h(an)f(ID.)210 707 y Fq(CSR)o(C)217 b Fs(\(input\))16
b(INTEGER)570 763 y(The)f(column)i(source)e(of)g(the)g(broadcast)f
(message)h(that)g(needs)h(an)f(ID.)150 876 y(Returns)h(a)g(BLA)o(CS)g
(message)g(ID)g(the)g(user)g(ma)o(y)f(safely)h(use)g(for)g(the)g
(destination)g(\(con)o(tributor\))f(of)150 932 y(a)g(primitiv)o(e)h
(broadcast)f(\(com)o(bine\).)1027 2828 y(33)p eop
%%Page: 34 38
34 37 bop 150 199 a Fr(References)173 303 y Fs([1])21
b(E.)14 b(Anderson,)h(Z.)f(Bai,)g(C.)g(Bisc)o(hof,)h(J.)f(Demmel,)h(J.)
f(Dongarra,)f(J.)h(Du)h(Croz,)e(A.)h(Green)o(baum,)243
360 y(S.)20 b(Hammarling,)i(A.)d(McKenney)l(,)j(S.)e(Ostrouc)o(ho)o(v,)
h(and)f(D.)f(Sorensen.)35 b Fm(\\LAP)m(A)o(CK)19 b(Users')243
416 y(Guide,)e(Se)n(c)n(ond)e(Edition")p Fs(.)20 b(SIAM,)15
b(Philadelphi)q(a,)i(P)l(A,)e(1995.)173 515 y([2])21
b(E.)14 b(Anderson,)h(Z.)e(Bai,)i(C.)e(Bisc)o(hof,)i(J.W.)f(Demmel,)g
(J.)g(J.)g(Dongarra,)f(J.)h(Du)g(Croz,)g(A.)g(Green-)243
571 y(baum,)h(S.)f(Hammarling,)g(A.)h(McKenney)l(,)g(and)g(A.)f
(McKenney)l(.)20 b(LAP)l(A)o(CK:)14 b(A)h(p)q(ortable)g(linear)243
628 y(algebra)f(library)h(for)e(high-p)q(erformance)h(computers.)k(T)l
(ec)o(hnical)d(Rep)q(ort)f(UT)g(CS-90-105,)e(LA-)243
684 y(P)l(A)o(CK)k(W)l(orking)f(Note)g(#20,)f(Univ)o(ersit)o(y)i(of)e
(T)l(ennessee,)i(1990.)173 783 y([3])21 b(M.)f(Barnett,)i(R.)f
(Little\014eld,)j(D.)c(P)o(a)o(yne,)h(and)g(R.)g(A.)g(v)m(an)g(de)g
(Geijn.)37 b(Global)21 b(com)o(bine)h(on)243 840 y(mesh)e(arc)o
(hitectures)h(with)f(w)o(ormhole)f(routing.)34 b(In)20
b Fm(7th)h(International)f(Par)n(al)r(lel)f(Pr)n(o)n(c)n(essing)243
896 y(Symp)n(osium)p Fs(,)c(1993.)173 995 y([4])21 b(J.)13
b(Choi,)f(J.)h(J.)f(Dongarra,)f(S.)h(Ostrouc)o(ho)o(v,)g(A.)g(P)l(.)g
(P)o(etitet,)g(D.)f(W.)h(W)l(alk)o(er,)g(and)h(R.)f(C.)g(Whaley)l(.)243
1052 y(The)i(Design)f(and)g(Implemen)o(tation)h(of)f(the)g(ScaLAP)l(A)o
(CK)h(LU,)f(QR,)h(and)f(Cholesky)g(F)l(actoriza-)243
1108 y(tion)h(Routines.)j Fm(T)m(o)c(app)n(e)n(ar)i(in)e(Scienti\014c)g
(Pr)n(o)n(gr)n(amming)p Fs(,)f(1994.)j(Also)e(a)o(v)m(ailable)i(as)d
(Univ)o(ersit)o(y)243 1164 y(of)j(T)l(ennessee)i(LAP)l(A)o(CK)e(W)l
(orking)h(Note)e(#80,)h(UT)g(CS-94-246,)e(1994.)173 1263
y([5])21 b(J.)16 b(Dongarra,)e(J.)h(Du)h(Croz,)f(I.)g(Du\013,)g(and)h
(S.)f(Hammarling.)22 b(\\A)15 b(Set)h(of)f(Lev)o(el)h(3)g(Basic)g
(Linear)243 1320 y(Algebra)23 b(Subprograms".)41 b Fm(A)o(CM)21
b(T)m(r)n(ansactions)g(on)i(Mathematic)n(al)g(Softwar)n(e)p
Fs(,)g(16\(1\):1{17,)243 1376 y(1990.)173 1475 y([6])e(J.)11
b(Dongarra,)e(J.)i(Du)f(Croz,)h(S.)f(Hammarling,)i(and)e(R.)h(Hanson.)h
(\\Algorithm)f(656:)16 b(An)11 b(extended)243 1532 y(Set)h(of)g(Basic)g
(Linear)h(Algebra)f(Subprograms:)18 b(Mo)q(del)13 b(Implemen)o(tation)f
(and)g(T)l(est)g(Programs".)243 1588 y Fm(A)o(CM)k(T)m(r)n(ansactions)e
(on)i(Mathematic)n(al)g(Softwar)n(e)p Fs(,)f(14\(1\):18{32,)c(1988.)173
1687 y([7])21 b(J.)h(Dongarra)f(and)h(E.)g(Grosse.)39
b(\\Distribution)23 b(of)f(mathematical)g(soft)o(w)o(are)e(via)i
(electronic)243 1743 y(mail".)f Fm(Communic)n(ations)15
b(of)i(the)f(A)o(CM)p Fs(,)e(30:403{407,)e(1987.)173
1842 y([8])21 b(J.)16 b(Dongarra)d(and)j(R.)f(v)m(an)h(de)g(Geijn.)k
(\\Tw)o(o)14 b(dimensional)k(Basic)e(Linear)g(Algebra)g(Comm)o(uni-)243
1899 y(cation)f(Subprograms".)j(T)l(ec)o(hnical)e(Rep)q(ort)e(UT)h
(CS-91-138,)e(LAP)l(A)o(CK)i(W)l(orking)f(Note)g(#37,)243
1955 y(Univ)o(ersit)o(y)i(of)f(T)l(ennessee,)h(1991.)173
2054 y([9])21 b(J.)g(Dongarra,)g(R.)g(v)m(an)h(de)f(Geijn,)i(and)e(D.)g
(W)l(alk)o(er.)37 b(\\A)21 b(Lo)q(ok)g(at)g(Scalable)i(Dense)e(Linear)
243 2111 y(Algebra)d(Librairies".)29 b(T)l(ec)o(hnical)19
b(Rep)q(ort)f(UT)g(CS-92-155,)e(LAP)l(A)o(CK)i(Working)g(Note)f(#43,)
243 2167 y(Univ)o(ersit)o(y)f(of)f(T)l(ennessee,)h(1992.)150
2266 y([10])21 b(Jac)o(k)e(J.)g(Dongarra,)f(Rob)q(ert)i(A.)e(v)m(an)i
(de)f(Geijn,)h(and)f(R.)g(Clin)o(t)h(Whaley)l(.)32 b(Tw)o(o)18
b(dimensional)243 2322 y(basic)i(linear)g(algebra)e(comm)o(unication)h
(subprograms.)30 b(In)19 b(Jac)o(k)f(J.)h(Dongarra)e(and)i(Bernard)243
2379 y(T)l(ouranc)o(heau,)12 b(editors,)g Fm(Envir)n(onments)g(and)g(T)
m(o)n(ols)g(for)h(Par)n(al)r(lel)f(Scienti\014c)f(Computing)p
Fs(,)h(pages)243 2435 y(31{40.)i(Elsevier)i(Science)h(Publishers)g
(B.V.,)e(1993.)150 2534 y([11])21 b(Message)c(P)o(assing)g(In)o
(terface)g(F)l(orum.)25 b(MPI:)17 b(A)g(Message-P)o(assing)g(In)o
(terface)g(Standard)g(.)26 b Fm(In-)243 2591 y(ternational)c(Journal)h
(of)f(Sup)n(er)n(c)n(omputer)h(Applic)n(ations)e(and)i(High)f
(Performanc)n(e)g(Comput-)243 2647 y(ing)p Fs(,)27 b(8\(3/4\),)e(1994.)
47 b(Sp)q(ecial)27 b(issue)f(on)e(MPI.)h(Also)g(a)o(v)m(ailable)i
(electronically)l(,)i(the)c(url)g(is)243 2704 y Fl
(ftp://www.netlib.org/mpi/mpi-)o(report.p)o(s)p Fs(.)1027
2828 y(34)p eop
%%Page: 35 39
35 38 bop 150 199 a Fs([12])21 b(A.)26 b(Geist,)j(A.)d(Beguelin,)31
b(J.)c(Dongarra,)g(W.)e(Jiang,)30 b(R.)c(Manc)o(hek,)j(and)e(V.)f
(Sunderam.)243 256 y Fm(PVM:)20 b(A)f(Users')g(Guide)h(and)g(T)m
(utorial)g(for)g(Networke)n(d)g(Par)n(al)r(lel)f(Computing)p
Fs(.)31 b(MIT)19 b(Press,)243 312 y(1994.)43 b(The)24
b(b)q(o)q(ok)f(is)h(a)o(v)m(ailable)h(electronically)l(,)j(the)23
b(url)h(is)g Fl(ftp://www.netlib.org/pvm)243 369 y(3/book/pvm-book.ps)p
Fs(.)150 462 y([13])d(G.)13 b(A.)h(Geist,)f(M.)g(T.)g(Heath,)h(B.)f(W.)
g(P)o(eyton,)g(and)h(P)l(.)f(H.)h(W)l(orley)l(.)j(\\a)c(users')h(guide)
g(to)f(PICL:)h(a)243 519 y(p)q(ortable)k(instrumen)o(ted)h(comm)o
(unication)f(library".)27 b(T)l(ec)o(hnical)19 b(Rep)q(ort)f
(ORNL/TM-11130,)243 575 y(Oak)e(Ridge)g(National)g(Lab)q(oratory)l(,)e
(1990.)150 669 y([14])21 b(Ching-Tien)g(Ho)d(and)h(S.)g(Lennart)g
(Johnsson.)31 b(Distributed)20 b(routing)e(algorithms)h(for)f(broad-)
243 726 y(casting)j(and)f(p)q(ersonalized)i(comm)o(unication)f(in)g(h)o
(yp)q(ercub)q(es.)36 b(In)21 b Fm(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)j
(the)g(1986)243 782 y(International)15 b(Confer)n(enc)n(e)g(on)h(Par)n
(al)r(lel)f(Pr)n(o)n(c)n(essing)p Fs(.)e(IEEE)i(Press,)g(1986.)150
876 y([15])21 b(C.)12 b(La)o(wson,)h(R.)f(Hanson,)h(D.)f(Kincaid,)j
(and)e(F.)e(Krogh.)16 b(\\Basic)d(Linear)g(Algebra)g(Subprograms)243
932 y(for)g(F)l(ortran)e(Usage".)k Fm(A)o(CM)e(T)m(r)n(ansactions)f(on)
i(Mathematic)n(al)g(Softwar)n(e)p Fs(,)f(5\(3\):308{323,)c(1979.)150
1026 y([16])21 b(R.)c(Clin)o(t)g(Whaley)l(.)24 b(\\Basic)17
b(Linear)h(Algebra)e(Comm)o(unication)h(Subprograms:)22
b(Analysis)c(and)243 1083 y(Implemen)o(tation)e(Across)d(Multiple)j(P)o
(arallel)f(Arc)o(hitectures".)k(T)l(ec)o(hnical)d(Rep)q(ort)e(UT)g
(CS-94-)243 1139 y(234,)g(LAP)l(A)o(CK)i(Working)f(Note)g(#73,)f(Univ)o
(ersit)o(y)i(of)f(T)l(ennessee,)h(1994.)1027 2828 y(35)p
eop
%%Page: 36 40
36 39 bop 150 199 a Fr(A)69 b(C)22 b(In)n(terface)h(to)g(the)f(BLA)n
(CS)p 598 299 905 2 v 598 309 V 597 366 2 57 v 606 366
V 631 349 a Fe(2)p Fs(is)p 720 366 V 50 w(Data)14 b(op)q(erated)i(on)f
(is)p 1269 366 V 154 w(TYPE)g(is)p 1493 366 V 1502 366
V 598 367 905 2 v 598 377 V 597 434 2 57 v 606 434 V
631 417 a(i)p 720 434 V 102 w(in)o(teger)p 1269 434 V
414 w(in)o(t)p 1493 434 V 1502 434 V 598 435 905 2 v
597 492 2 57 v 606 492 V 631 475 a(s)p 720 492 V 96 w(single)i
(precision)g(real)p 1269 492 V 159 w(\015oat)p 1493 492
V 1502 492 V 598 493 905 2 v 597 550 2 57 v 606 550 V
631 533 a(d)p 720 550 V 89 w(double)g(precision)g(real)p
1269 550 V 139 w(double)p 1493 550 V 1502 550 V 598 551
905 2 v 597 608 2 57 v 606 608 V 631 591 a(c)p 720 608
V 94 w(single)g(precision)g(complex)p 1269 608 V 70 w(\015oat)p
1493 608 V 1502 608 V 598 610 905 2 v 597 666 2 57 v
606 666 V 631 649 a(z)p 720 666 V 94 w(double)g(precision)g(complex)p
1269 666 V 50 w(double)p 1493 666 V 1502 666 V 598 668
905 2 v 598 678 V 587 755 a(T)l(able)f(6:)j(Pre\014x)d(to)e(C)h(t)o(yp)
q(e)h(declaration)f(mapping)150 872 y(Throughout)d(this)h(guide)g(w)o
(e)f(ha)o(v)o(e)g(b)q(een)h(presen)o(ting)g(the)g(F)l(ortran)e(77)h(in)
o(terface)g(to)g(the)g(BLA)o(CS.)h(The)150 929 y(BLA)o(CS)j(also)f(ha)o
(v)o(e)f(a)h(C)g(in)o(terface.)20 b(In)c(order)f(to)f(a)o(v)o(oid)h
(name-space)g(con\015icts,)h(all)g(C)f(BLA)o(CS)g(ha)o(v)o(e)150
985 y(C)i(prep)q(ended)i(to)e(them)g(\(th)o(us)g(DGEBS2D)g(b)q(ecomes)h
(Cdgebs2d,)f(for)g(instance\).)27 b(The)17 b(only)h(other)150
1042 y(di\013erence)g(is)g(that)e(C)h(allo)o(ws)g(the)g(user)g(to)f
(pass)h(parameters)f(b)o(y)h(v)m(alue)h(or)f(b)o(y)g(address,)g(and)g
(the)g(C)150 1098 y(in)o(terface)f(tak)o(es)e(adv)m(an)o(tage)h(of)g
(this)g(capabilit)o(y)l(.)22 b(The)16 b(names)f(and)g(uses)h(of)f(the)g
(parameters)g(remain)150 1155 y(the)h(same.)21 b(In)c(particular,)f
(the)g(user)g(should)h(note)e(that)g(the)h(C)g(in)o(terface)g(BLA)o(CS)
g(still)h(require)g(the)150 1211 y(arra)o(ys)d(passed)h(in)h(to)f(use)g
(column-ma)s(jor)g(storage.)221 1269 y(The)e(calling)i(sequences)f(and)
g(parameter)e(declarations)i(of)f(the)g(C)g(BLA)o(CS)h(are)f(presen)o
(ted)h(b)q(elo)o(w.)150 1325 y(Here)f(w)o(e)g(use)g(the)g
Fe(2)g Fs(to)f(represen)o(t)h(the)f(t)o(yp)q(e)h(pre\014x.)20
b(T)l(able)14 b(6)e(pro)o(vides)h(the)g(mapping)h(from)e(the)h(t)o(yp)q
(e)150 1382 y(pre\014x)19 b(to)e(the)h(TYPE)g(declaration.)29
b(This)19 b(information,)g(coupled)g(with)g(the)f(routine)h
(descriptions)150 1438 y(giv)o(en)d(in)g(the)f(b)q(o)q(dy)h(of)f(this)g
(rep)q(ort,)g(should)h(allo)o(w)f(the)h(C)f(programmer)f(to)g(use)i
(the)f(BLA)o(CS.)150 1566 y Fk(A.1)56 b(Supp)r(ort)18
b(Routines)150 1654 y Fq(A.1.1)51 b(Initializat)q(ion)150
1741 y Fs(v)o(oid)15 b(Cblacs)p 382 1741 14 2 v 17 w(pinfo)72
b(\()15 b(in)o(t)g(*m)o(ypn)o(um)p 692 1757 173 2 v(,)g(in)o(t)g(*npro)
q(cs)p 985 1757 131 2 v 16 w(\))150 1801 y(v)o(oid)g(Cblacs)p
382 1801 14 2 v 17 w(setup)65 b(\()15 b(in)o(t)g(*m)o(ypn)o(um)p
692 1816 173 2 v(,)g(in)o(t)g(*npro)q(cs)p 985 1816 131
2 v 16 w(\))150 1860 y(v)o(oid)g(Cblacs)p 382 1860 14
2 v 17 w(get)110 b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)h(what,)e(in)o
(t)h(*v)m(al)p 1124 1867 57 2 v 16 w(\))150 1919 y(v)o(oid)g(Cblacs)p
382 1919 14 2 v 17 w(set)115 b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)h
(what,)e(in)o(t)h(*v)m(al)h(\))150 1979 y(v)o(oid)f(Cblacs)p
382 1979 V 17 w(gridinit)26 b(\()15 b(in)o(t)g(*icon)o(txt)p
692 1986 139 2 v(,)g(c)o(har)g(*order,)f(in)o(t)h(npro)o(w,)g(in)o(t)g
(np)q(col)i(\))150 2038 y(v)o(oid)e(Cblacs)p 382 2038
14 2 v 17 w(gridmap)6 b(\()16 b(in)o(t)f(*icon)o(txt)p
692 2045 139 2 v(,)g(in)o(t)g(*pmap,)g(in)o(t)g(ldpmap,)h(in)o(t)g
(npro)o(w,)e(in)o(t)h(np)q(col)i(\))150 2165 y Fq(A.1.2)51
b(Destruction)150 2252 y Fs(v)o(oid)15 b(Cblacs)p 382
2252 14 2 v 17 w(freebu\013)22 b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)
h(w)o(ait)e(\))150 2311 y(v)o(oid)h(Cblacs)p 382 2311
V 17 w(gridexit)k(\()c(in)o(t)g(icon)o(txt)g(\))150 2371
y(v)o(oid)g(Cblacs)p 382 2371 V 17 w(ab)q(ort)63 b(\()15
b(in)o(t)g(icon)o(txt,)g(in)o(t)h(errorn)o(um)e(\))150
2430 y(v)o(oid)h(Cblacs)p 382 2430 V 17 w(exit)97 b(\()15
b(in)o(t)g(done\015ag)h(\))150 2557 y Fq(A.1.3)51 b(Informational)19
b(and)f(Miscellaneous)150 2644 y Fs(v)o(oid)d(Cblacs)p
382 2644 V 17 w(gridinfo)k(\()c(in)o(t)g(icon)o(txt,)g(in)o(t)h(*npro)o
(w)p 929 2659 123 2 v -1 w(,)f(in)o(t)g(*np)q(col)p 1171
2659 108 2 v 1 w(,)g(in)o(t)g(*m)o(ypro)o(w)p 1398 2659
158 2 v -1 w(,)g(in)o(t)g(*m)o(yp)q(col)p 1676 2659 143
2 v 16 w(\))150 2703 y(in)o(t)46 b(Cblacs)p 384 2703
14 2 v 17 w(pn)o(um)p 250 2719 261 2 v 57 w(\()15 b(in)o(t)g(icon)o
(txt,)g(in)o(t)h(pro)o(w,)e(in)o(t)h(p)q(col)h(\))1027
2828 y(36)p eop
%%Page: 37 41
37 40 bop 150 199 a Fs(v)o(oid)15 b(Cblacs)p 382 199
14 2 v 17 w(p)q(co)q(ord)35 b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)h
(pn)o(um,)f(in)o(t)g(*pro)o(w)p 1138 215 98 2 v -1 w(,)g(in)o(t)g(*p)q
(col)p 1355 215 83 2 v 16 w(\))150 259 y(v)o(oid)g(Cblacs)p
382 259 14 2 v 17 w(barrier)37 b(\()15 b(icon)o(txt,)g(c)o(har)g(*scop)
q(e)g(\))150 379 y Fq(A.1.4)51 b(Uno\016cial)150 465
y Fs(v)o(oid)61 b(Csetp)o(vm)o(tids\()14 b(in)o(t)i(n)o(tasks,)e(in)o
(t)h(*tids)h(\))150 524 y(double)g(Cdcputime)25 b(\()15
b(\))150 583 y(double)h(Cdw)o(alltime)g(\()f(\))150 643
y(in)o(t)91 b(Cksendid)60 b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)h
(rdest,)e(in)o(t)i(cdest)f(\))150 702 y(in)o(t)91 b(Ckrecvid)66
b(\()15 b(in)o(t)g(icon)o(txt,)g(in)o(t)h(rsrc,)e(in)o(t)i(csrc)f(\))
150 762 y(in)o(t)91 b(Ckbsid)105 b(\()15 b(in)o(t)g(icon)o(txt,)g(c)o
(har)g(*scop)q(e)g(\))150 821 y(in)o(t)91 b(Ckbrid)105
b(\()15 b(in)o(t)g(icon)o(txt,)g(c)o(har)g(*scop)q(e,)g(in)o(t)h(rsrc,)
e(in)o(t)h(csrc)h(\))150 943 y Fk(A.2)56 b(P)n(oin)n(t)19
b(to)g(P)n(oin)n(t)150 1029 y Fs(v)o(oid)c(C)p Fe(2)p
Fs(gesd2d\()g(in)o(t)h(icon)o(txt,)e(in)o(t)i(m,)e(in)o(t)i(n,)f(TYPE)g
(*A,)f(in)o(t)i(lda,)f(in)o(t)h(rdest,)e(in)o(t)i(cdest)f(\))150
1088 y(v)o(oid)g(C)p Fe(2)p Fs(gerv2d\()g(in)o(t)g(icon)o(txt,)g(in)o
(t)g(m,)g(in)o(t)g(n,)g(TYPE)g(*A)p 1141 1095 35 2 v(,)g(in)o(t)g(lda,)
h(in)o(t)f(rsrc,)g(in)o(t)g(csrc)h(\))150 1148 y(v)o(oid)f(C)p
Fe(2)p Fs(trsd2d)5 b(\()15 b(in)o(t)c(icon)o(txt,)g(c)o(har)g(*uplo,)g
(c)o(har)g(*diag,)g(in)o(t)g(m,)g(in)o(t)g(n,)h(TYPE)e(*A,)h(in)o(t)g
(lda,)g(in)o(t)h(rdest,)480 1204 y(in)o(t)j(cdest)h(\))150
1264 y(v)o(oid)f(C)p Fe(2)p Fs(trrv2d)7 b(\()14 b(in)o(t)j(icon)o(txt,)
g(c)o(har)g(*uplo,)h(c)o(har)f(*diag,)g(in)o(t)g(in)o(t)h(n,)f(TYPE)g
(*A)p 1566 1271 V(,)g(in)o(t)g(lda,)h(in)o(t)g(rsrc,)480
1320 y(in)o(t)d(csrc)h(\))150 1442 y Fk(A.3)56 b(Broadcasts)150
1528 y Fs(v)o(oid)15 b(C)p Fe(2)p Fs(gebs2d\()g(in)o(t)h(icon)o(txt,)e
(c)o(har)h(*scop)q(e,)g(c)o(har)g(*top,)f(in)o(t)i(m,)e(in)o(t)i(n,)f
(TYPE)g(*A,)f(in)o(t)i(lda)g(\))150 1587 y(v)o(oid)f(C)p
Fe(2)p Fs(gebr2d\()g(in)o(t)d(icon)o(txt,)h(c)o(har)e(*scop)q(e,)i(c)o
(har)f(*top,)g(in)o(t)g(m,)g(in)o(t)g(n,)h(TYPE)f(*A)p
1586 1594 V(,)g(in)o(t)g(lda,)h(in)o(t)g(rsrc,)480 1644
y(in)o(t)i(csrc)h(\))150 1703 y(v)o(oid)f(C)p Fe(2)p
Fs(trbs2d)5 b(\()15 b(in)o(t)k(icon)o(txt,)f(c)o(har)h(*scop)q(e,)g(c)o
(har)f(*top,)g(c)o(har)g(*uplo,)i(c)o(har)e(*diag,)h(in)o(t)g(m,)f(in)o
(t)h(n,)480 1759 y(TYPE)c(*A,)g(in)o(t)g(lda)h(\))150
1819 y(v)o(oid)f(C)p Fe(2)p Fs(trbr2d)5 b(\()15 b(in)o(t)e(icon)o(txt,)
f(c)o(har)h(*uplo,)g(c)o(har)f(*diag,)h(in)o(t)g(m,)f(in)o(t)h(n,)g
(TYPE)f(*A)p 1584 1826 V(,)h(in)o(t)g(lda,)g(in)o(t)g(rsrc,)480
1875 y(in)o(t)i(csrc)h(\))150 1997 y Fk(A.4)56 b(Com)n(bines)150
2083 y Fs(v)o(oid)15 b(C)p Fe(2)p Fs(gsum2d\()g(in)o(t)23
b(icon)o(txt,)h(c)o(har)f(*scop)q(e,)h(c)o(har)f(*top,)h(in)o(t)f(m,)h
(in)o(t)f(n,)i(TYPE)d(*A)p 1728 2090 V(,)j(in)o(t)e(lda,)480
2139 y(in)o(t)15 b(rdest,)g(in)o(t)h(cdest)f(\))150 2199
y(v)o(oid)g(C)p Fe(2)p Fs(gamx2d\()f(in)o(t)23 b(icon)o(txt,)h(c)o(har)
f(*scop)q(e,)h(c)o(har)e(*top,)i(in)o(t)f(m,)g(in)o(t)g(n,)i(TYPE)d(*A)
p 1729 2206 V(,)i(in)o(t)f(lda,)480 2255 y(in)o(t)15
b(*RA)p 572 2262 68 2 v 1 w(,)g(in)o(t)g(*CA)p 760 2262
67 2 v(,)f(in)o(t)i(R)o(C\015ag,)e(in)o(t)i(rdest,)e(in)o(t)i(cdest)f
(\))150 2315 y(v)o(oid)g(C)p Fe(2)p Fs(gamn2d\()g(in)o(t)22
b(icon)o(txt,)i(c)o(har)f(*scop)q(e,)h(c)o(har)e(*top,)h(in)o(t)g(m,)h
(in)o(t)f(n,)h(TYPE)e(*A)p 1729 2322 35 2 v(,)i(in)o(t)f(lda,)480
2371 y(in)o(t)15 b(*RA)p 572 2378 68 2 v 1 w(,)g(in)o(t)g(*CA)p
760 2378 67 2 v(,)f(in)o(t)i(R)o(C\015ag,)e(in)o(t)i(rdest,)e(in)o(t)i
(cdest)f(\))1027 2828 y(37)p eop
%%Page: 38 42
38 41 bop 150 199 a Fr(B)69 b(Degrees)22 b(of)h(Blo)r(c)n(king)150
301 y Fs(This)17 b(app)q(endix)h(pro)o(vides)e(further)g(discussion)i
(of)e(the)g(blo)q(c)o(king)i(lev)o(els)f(in)o(tro)q(duced)g(in)g
(Section)g(4.5,)150 357 y(and)g(the)g(reasons)f(b)q(ehind)j(the)e(c)o
(hoices)h(for)e(the)h(BLA)o(CS)g(blo)q(c)o(king)h(lev)o(els.)26
b(The)17 b(blo)q(c)o(king)h(lev)o(els)g(of)150 414 y(the)d(BLA)o(CS)h
(comm)o(unication)g(routines)f(are:)218 506 y Fn(\017)23
b Fl(vXXSD2D)14 b Fs(\()p Fl(XX)g Fs(=)i Fl(GE)p Fs(,)e(or)h
Fl(TR)p Fs(\))g(:)f(Lo)q(cally-blo)q(c)o(ki)q(ng.)218
600 y Fn(\017)23 b Fl(vXXRV2D)14 b Fs(\()p Fl(XX)g Fs(=)i
Fl(GE)p Fs(,)e(or)h Fl(TR)p Fs(\))g(:)f(Globally-blo)q(c)o(king.)218
693 y Fn(\017)23 b Fl(vXXBS2D)14 b Fs(\()p Fl(XX)g Fs(=)i
Fl(GE)p Fs(,)e(or)h Fl(TR)p Fs(\))g(:)f(Globally-blo)q(c)o(king.)218
786 y Fn(\017)23 b Fl(vXXBR2D)14 b Fs(\()p Fl(XX)g Fs(=)i
Fl(GE)p Fs(,)e(or)h Fl(TR)p Fs(\))g(:)f(Globally-blo)q(c)o(king.)218
879 y Fn(\017)23 b Fl(vGZZZ2D)14 b Fs(\()p Fl(ZZZ)g Fs(=)i
Fl(SUM)p Fs(,)e Fl(AMX)h Fs(or)g Fl(AMN)p Fs(\))f(:)h(Globally-blo)q(c)
o(king.)150 1001 y Fk(B.1)55 b(Non-blo)r(c)n(king)18
b(comm)n(unication)150 1087 y Fs(The)d(BLA)o(CS)h(do)f(not)g(pro)o
(vide)h(the)f(user)g(with)h(an)o(y)e(non-blo)q(c)o(king)j(routines.)k
(An)15 b(understanding)h(of)150 1143 y(this)e(t)o(yp)q(e)g(of)g(comm)o
(unication)h(should)g(still)g(b)q(e)g(helpful)h(to)d(the)h(user,)g(ho)o
(w)o(ev)o(er.)k(Therefore,)c(w)o(e)g(shall)150 1200 y(brie\015y)i
(discuss)f(non-blo)q(c)o(king)h(comm)o(unication,)f(and)g(wh)o(y)f(the)
h(BLA)o(CS)g(do)f(not)g(explicitly)j(pro)o(vide)150 1256
y(this)f(capabilit)o(y)l(.)221 1313 y(As)c(previously)i(men)o(tioned,)g
(after)e(a)g(non-blo)q(c)o(king)j(comm)o(unication)e(has)g(b)q(een)h(p)
q(osted,)f(the)g(user)150 1369 y(m)o(ust)i(prob)q(e)g(to)g(determine)h
(when)f(the)h(op)q(eration)f(has)g(completed.)21 b(This)15
b(allo)o(ws)h(the)f(user)g(to)g(b)q(egin)150 1425 y(an)j(op)q(eration,)
i(and)e(then)h(do)g(unrelated)g(w)o(ork)f(un)o(til)h(the)g(op)q
(eration)g(completes)g(\(whic)o(h)g(is)g(deter-)150 1482
y(mined)h(b)o(y)e(the)h(aforemen)o(tioned)f(probing\).)30
b(In)19 b(some)f(relativ)o(ely)i(restricted)f(conditions,)h(this)f(can)
150 1538 y(lead)h(to)f(fairly)h(large)g(p)q(erformance)g(gains.)33
b(Ho)o(w)o(ev)o(er,)19 b(w)o(e)g(ha)o(v)o(e)h(found)g(the)f(use)h(of)f
(non-blo)q(c)o(king)150 1595 y(comm)o(unication)d(to)f(b)q(e)h(highly)h
(error-prone,)e(often)g(leading)i(to)e(non-deterministic)j(or)d(o)o(v)o
(erly)g(com-)150 1651 y(plex)21 b(co)q(de.)35 b(Because)21
b(of)f(the)g(di\016cult)o(y)h(in)g(correctly)f(using)h(non-blo)q(c)o
(king)h(comm)o(unication,)g(the)150 1708 y(BLA)o(CS,)d(whic)o(h)h(try)f
(to)f(pro)o(vide)i(an)f(easy-to-program)f(in)o(terface,)i(do)f(not)g
(explicitly)j(supp)q(ort)d(it.)150 1764 y(Ho)o(w)o(ev)o(er,)f(whenev)o
(er)g(it)h(yields)h(increased)f(p)q(erformance)g(or)f(functionalit)o(y)
l(,)i(the)e(BLA)o(CS)h(ma)o(y)f(use)150 1821 y(non-blo)q(c)o(king)d
(comm)o(unication)e(in)o(ternally)l(,)i(where)e(its)g(complexit)o(y)h
(can)f(b)q(e)h(shielded)h(from)d(the)h(user.)221 1877
y(T)l(o)k(giv)o(e)h(the)g(reader)g(an)g(idea)g(of)g(ho)o(w)f(non-blo)q
(c)o(king)i(comm)o(unication)g(w)o(orks)e(w)o(e)g(will)j(use)e(the)150
1934 y(example)c(of)f(t)o(w)o(o)f(pro)q(cesses)h(\(for)g(con)o(v)o
(enience)h(lab)q(eled)i(pro)q(cess)d(0)g(and)h(pro)q(cess)f(1\))g(exc)o
(hanging)h(data)150 1990 y(\(the)j(user)h(bu\013er)g(is)g(pro)o(vided)g
(b)o(y)g(the)f(v)m(ariable)i Fl(X)p Fs(\).)e(This)h(example)h(will)g(b)
q(e)f(used)g(in)h(eac)o(h)e(section)150 2046 y(to)f(illustrate)i(ho)o
(w)e(the)h(v)m(arious)g(lev)o(els)h(of)f(blo)q(c)o(king)h(w)o(ork.)23
b(The)17 b(follo)o(wing)h(pseudo-co)q(de)g(fragmen)o(t)150
2103 y(sho)o(ws)d(a)f(p)q(ossible)j(w)o(a)o(y)e(to)f(p)q(erform)h(this)
h(sw)o(ap:)222 2195 y Fl(IAM)23 b(=)h(MYPROCESSID\(\))222
2252 y(IF\()f(IAM.EQ.0)g(\))h(THEN)293 2308 y(NON_BLOCKING_SEND)e(FROM)
h(VARIABLE)g(X)h(TO)f(PROCESS)g(1)293 2365 y(NON_BLOCKING_RECV)f(INTO)h
(VARIABLE)g(TMP)g(FROM)h(PROCESS)e(1)222 2421 y(ELSE)h(IF\()g(IAM)h
(.EQ.)f(1)h(\))g(THEN)293 2478 y(NON_BLOCKING_SEND)e(FROM)h(VARIABLE)g
(X)h(TO)f(PROCESS)g(0)293 2534 y(NON_BLOCKING_RECV)f(INTO)h(VARIABLE)g
(TMP)g(FROM)h(PROCESS)e(0)222 2591 y(END)h(IF)222 2647
y(.)222 2704 y(.)1027 2828 y Fs(38)p eop
%%Page: 39 43
39 42 bop 222 199 a Fl(<DO)23 b(UNRELATED)g(WORK>)222
256 y(.)222 312 y(.)222 369 y(PROBE)g(UNTIL)g(SEND)g(COMPLETES)222
425 y(PROBE)g(UNTIL)g(RECV)g(COMPLETES)222 482 y(X)g(=)h(TMP)221
577 y Fs(Ev)o(en)e(with)h(this)g(simple)h(example,)h(w)o(e)d(can)h(see)
g(some)f(opp)q(ortunities)i(for)e(user)g(error.)42 b(F)l(or)150
633 y(example,)19 b(sa)o(y)f(the)g(user)g(forgets)f(to)h(prob)q(e)g
(for)g(completion)h(of)f(the)g(send.)29 b(Then,)19 b(let)g(us)f(sa)o(y)
g(that)150 690 y(pro)q(cess)j(0's)f(receiv)o(e)i(completes)f(quic)o
(kly)l(.)38 b(Therefore,)22 b(w)o(e)e(o)o(v)o(erwrite)g(X)h(with)g(pro)
q(cess)g(1's)f(data.)150 746 y(Then,)e(when)g(the)g(send)g(is)g
(actually)g(completed,)h(pro)q(cess)f(1)f(receiv)o(es)h(its)g(o)o(wn)f
(data)g(bac)o(k,)g(instead)150 803 y(of)e(pro)q(cess)h(0's)f(data.)21
b(There)16 b(are)g(man)o(y)f(other)h(w)o(a)o(ys)e(to)h(go)h(wrong)f
(here,)h(and)g(it)g(is)g(for)f(this)h(reason)150 859
y(that)e(the)i(BLA)o(CS)f(pro)o(vide)h(no)f(explicit)j(non-blo)q(c)o
(king)f(op)q(erations.)150 983 y Fk(B.2)55 b(Lo)r(cally-blo)r(c)n(king)
150 1070 y Fs(As)19 b(men)o(tioned)i(b)q(efore,)f(only)g(send)g(op)q
(erations)f(ma)o(y)g(b)q(e)h(lo)q(cally-blo)q(c)o(ki)q(ng.)35
b(Remem)o(b)q(er)20 b(that)f(re-)150 1126 y(turning)c(from)g(a)f(lo)q
(cally-blo)q(c)o(ki)q(ng)k(op)q(eration)d(implies)i(the)e(bu\013er)g
(is)g(a)o(v)m(ailable)i(for)d(re-use,)h(and)g(will)150
1183 y(complete)j(ev)o(en)g(if)g(the)g(complemen)o(t)g(of)f(the)h
(routine)g(has)g(not)f(b)q(een)i(p)q(osted.)27 b(It)18
b(is)g(imp)q(ossible)i(for)150 1239 y(a)d(receiv)o(e)h(to)f(store)f
(the)i(send's)f(con)o(ten)o(ts)g(in)h(the)f(bu\013er)g(\(and)g(th)o(us)
g(free)h(it)f(for)g(re-use\))g(b)q(efore)h(the)150 1295
y(message)g(has)h(b)q(een)h(sen)o(t,)f(and)g(therefore)f(w)o(e)h(see)g
(that)f(receiv)o(es)h(ma)o(y)f(only)i(b)q(e)f(non-)g(or)g(globally-)150
1352 y(blo)q(c)o(king.)221 1409 y(W)l(e)g(sa)o(y)g(that)g(a)g(lo)q
(cally-blo)q(c)o(ki)q(ng)j(send)f(guaran)o(tees)d(completion)j(ev)o(en)
f(if)g(the)g(corresp)q(onding)150 1465 y(receiv)o(e)13
b(has)f(not)f(b)q(een)i(called.)21 b(This)12 b(is)g(actually)h(to)q(o)e
(strong)g(an)h(assertion.)19 b(In)12 b(realit)o(y)l(,)h(w)o(e)f(guaran)
o(tee)150 1522 y(completion)k(up)g(to)f(the)g(limit)h(of)f(a)o(v)m
(ailable)i(bu\013er)e(space.)221 1578 y(If)g(a)h(lo)q(cally-blo)q(c)o
(king)i(send)e(is)h(b)q(egun,)f(and)f(the)h(corresp)q(onding)g(receiv)o
(e)h(has)e(not)g(b)q(een)i(p)q(osted,)150 1635 y(the)c(data)g(to)g(b)q
(e)h(sen)o(t)f(m)o(ust)f(b)q(e)i(bu\013ered)g(so)f(that)g(is)g(not)g
(lost)h(when)f(the)h(user's)f(bu\013er)g(is)h(returned)g(to)150
1691 y(him.)22 b(This)16 b(system)f(bu\013er)h(ma)o(y)f(b)q(e)h(allo)q
(cated)h(b)o(y)f(the)f(sending)i(pro)q(cess,)f(the)g(destination)g(pro)
q(cess,)150 1748 y(or)j(b)o(y)g(the)h(hardw)o(are)e(whic)o(h)i(transp)q
(orts)f(the)g(send.)33 b(In)20 b(an)o(y)f(case,)h(the)g(amoun)o(t)e(of)
h(bu\013er)h(space)150 1804 y(a)o(v)m(ailable)13 b(will)f(alw)o(a)o(ys)
e(ha)o(v)o(e)h(an)g(upp)q(er)g(limit)i(\(amoun)o(t)c(of)i(ph)o(ysical)h
(memory)e(free,)i(amoun)o(t)e(of)g(virtual)150 1861 y(memory)l(,)20
b(etc.\).)34 b(When)20 b(this)h(space)f(is)h(exhausted,)g(the)f(sends)h
(will)g(blo)q(c)o(k)g(un)o(til)g(enough)f(system)150
1917 y(bu\013er)e(space)g(b)q(ecomes)h(a)o(v)m(ailable)g(\(i.e.,)f(un)o
(til)h(enough)f(of)g(the)g(outstanding)g(sends)g(are)g(completed)150
1974 y(b)o(y)e(the)f(p)q(osting)h(of)f(the)h(corresp)q(onding)h(receiv)
o(e\).)k(Therefore,)16 b(while)h(lo)q(cally-blo)q(c)o(king)i(sends)d
(allo)o(w)150 2030 y(us)f(greater)f(\015exibilit)o(y)j(than)e(the)g
(more)f(restrictiv)o(e)i(globally-blo)q(c)o(king)h(sends)f(w)o(e)e
(will)j(so)q(on)d(discuss,)150 2087 y(it)h(is)h(not)f(a)g(go)q(o)q(d)g
(idea)h(to)e(p)q(ost)h(a)g(great)g(n)o(um)o(b)q(er)g(of)g(them)g
(without)g(p)q(osting)h(an)o(y)f(receiv)o(es.)221 2143
y(As)e(in)h(the)f(non-blo)q(c)o(king)i(section,)f(w)o(e)f(giv)o(e)h(an)
f(example)h(of)f(t)o(w)o(o)f(pro)q(cesses)h(sw)o(apping)h(data,)f(this)
150 2200 y(time)j(using)f(a)g(lo)q(cally-blo)q(c)o(ki)q(ng)j(send,)d
(and)h(a)f(globally-blo)q(c)o(king)j(receiv)o(e.)222
2308 y Fl(IAM)23 b(=)h(MYPROCESSID\(\))222 2365 y(IF\()f(IAM.EQ.0)g(\))
h(THEN)293 2421 y(LOCALLY_BLOCKING_SEND)45 b(FROM)23
b(VARIABLE)g(X)h(TO)f(PROCESS)g(1)293 2478 y(GLOBALLY_BLOCKING_RECV)e
(INTO)i(VARIABLE)g(X)h(FROM)f(PROCESS)g(1)222 2534 y(ELSE)g(IF\()g(IAM)
h(.EQ.)f(1)h(\))g(THEN)293 2591 y(LOCALLY_BLOCKING_SEND)45
b(FROM)23 b(VARIABLE)g(X)h(TO)f(PROCESS)g(0)293 2647
y(GLOBALLY_BLOCKING_RECV)e(INTO)i(VARIABLE)g(X)h(FROM)f(PROCESS)g(0)222
2704 y(END)g(IF)1027 2828 y Fs(39)p eop
%%Page: 40 44
40 43 bop 150 199 a Fk(B.3)55 b(Globally-blo)r(c)n(king)150
289 y Fs(Our)18 b(de\014nition)i(of)d(globally-blo)q(c)o(king)j(is)f(a)
e(little)i(lo)q(oser)f(than)g(the)f(common)h(de\014nition.)29
b(Usually)l(,)150 346 y(globally-blo)q(c)o(king)c(means)c(that)g
(return)h(from)f(an)h(op)q(eration)g(insures)h(that)e(the)h(complemen)o
(tary)150 402 y(op)q(eration)16 b(has)g(also)f(b)q(een)i(called.)24
b(In)16 b(other)f(w)o(ords,)g(return)h(from)f(a)g(globally-blo)q(c)o
(king)k(send)d(guar-)150 458 y(an)o(tees)i(that)g(the)h(complemen)o
(tary)f(receiv)o(e)i(has)e(b)q(een)i(p)q(osted.)30 b(In)19
b(this)g(do)q(cumen)o(t,)h(ho)o(w)o(ev)o(er,)e(the)150
515 y(term)12 b(globally-blo)q(c)o(king)k(is)d(used)g(to)f(describ)q(e)
i(an)o(y)e(op)q(eration)h(that)f(is)h(not)f(guaran)o(teed)g(\(see)h
(the)g(pre-)150 571 y(vious)i(section)g(for)g(a)f(little)i(hedging)g
(on)e(using)i(the)f(w)o(ord)f(\\guaran)o(teed")f(in)j(this)f(con)o
(text\))f(to)g(return)150 628 y(without)k(the)g(complemen)o(tary)g(p)q
(ost.)27 b(This)19 b(means,)f(for)f(instance,)i(that)e(while)i(w)o(e)f
(m)o(ust)f(program)150 684 y(our)f(co)q(de)h(to)f(allo)o(w)h(for)f(the)
g(p)q(ossibilit)o(y)j(that)d(a)g(globally-blo)q(c)o(king)k(send)d(do)q
(es)f(not)h(complete)g(un)o(til)150 741 y(the)c(corresp)q(onding)i
(receiv)o(e)f(is)g(p)q(osted,)g(w)o(e)f(cannot)g(assume)g(that)g
(returning)h(from)e(the)i(send)g Fm(implies)150 797 y
Fs(that)h(the)h(receiv)o(e)h(has)f(b)q(een)h(p)q(osted.)23
b(F)l(or)15 b(example,)i(w)o(e)e(can't)h(use)g(one)g(globally-blo)q(c)o
(king)j(send)e(to)150 854 y(sync)o(hronize)f(t)o(w)o(o)e(pro)q(cesses.)
221 912 y(The)h(familiar)g(sw)o(apping)h(example,)f(this)g(time)g
(using)h(globally-blo)q(c)o(king)i(sends)d(and)g(receiv)o(es,)g(is)150
969 y(giv)o(en)h(b)q(elo)o(w.)222 1073 y Fl(IAM)23 b(=)h
(MYPROCESSID\(\))222 1129 y(IF\()f(IAM.EQ.0)g(\))h(THEN)293
1186 y(GLOBALLY_BLOCKING_SEND)d(FROM)i(VARIABLE)g(X)h(TO)f(PROCESS)g(1)
293 1242 y(GLOBALLY_BLOCKING_RECV)e(INTO)i(VARIABLE)g(X)h(FROM)f
(PROCESS)g(1)222 1299 y(ELSE)g(IF\()g(IAM)h(.EQ.)f(1)h(\))g(THEN)293
1355 y(TMP)g(=)f(X)293 1412 y(GLOBALLY_BLOCKING_RECV)e(INTO)i(VARIABLE)
g(X)h(FROM)f(PROCESS)g(0)293 1468 y(GLOBALLY_BLOCKING_SEND)e(FROM)i
(VARIABLE)g(TMP)h(TO)f(PROCESS)g(0)222 1525 y(END)g(IF)221
1627 y Fs(Note)12 b(that)h(pro)q(cess)g(0)g(p)q(osts)g(a)g(send,)h
(follo)o(w)o(ed)g(b)o(y)f(a)g(receiv)o(e,)h(while)g(pro)q(cess)g(1)f(p)
q(osts)g(the)g(receiv)o(e)150 1683 y(\014rst,)k(and)g(then)g(the)g
(send.)26 b(If)17 b(w)o(e)f(attempted)h(to)f(use)h(the)g(same)g
(pattern)g(as)f(w)o(e)h(did)h(with)f(lo)q(cally-)150
1740 y(blo)q(c)o(king)h(sends)g(\(i.e.,)e(b)q(oth)h(pro)q(cesses)h
(send)f(and)g(then)h(receiv)o(e\),)f(with)g(a)g(globally-blo)q(c)o
(king)j(send,)150 1796 y(w)o(e)f(see)g(that)g(pro)q(cess)g(0)g(en)o
(ters)f(the)i(send,)g(and)f(w)o(aits)g(for)f(pro)q(cess)h(1)g(to)g
(start)e(its)j(receiv)o(e)g(b)q(efore)150 1853 y(con)o(tin)o(uing.)32
b(In)20 b(the)f(mean)o(time,)h(pro)q(cess)f(1)g(starts)e(to)i(send)g
(to)g(0,)g(and)g(therefore)g(w)o(aits)g(for)f(0)h(to)150
1909 y(receiv)o(e)d(b)q(efore)g(con)o(tin)o(uing.)22
b(Both)15 b(pro)q(cesses)h(are)f(no)o(w)g(w)o(aiting)g(on)h(eac)o(h)f
(other,)g(and)h(the)f(program)150 1965 y(will)i(therefore)e(nev)o(er)g
(con)o(tin)o(ue.)221 2024 y(F)l(or)k(this)h(simple)h(sw)o(ap)e
(example,)i(ha)o(ving)f(one)f(pro)q(cess)h(rev)o(erse)f(the)h(order)f
(of)h(its)f(calls)i(is)f(an)150 2080 y(ob)o(vious)i(\014x.)38
b(Ho)o(w)o(ev)o(er,)21 b(when)h(the)g(comm)o(unication)g(is)f(not)g
(just)g(b)q(et)o(w)o(een)h(t)o(w)o(o)e(pro)q(cesses,)j(but)150
2137 y(rather)13 b(in)o(v)o(olv)o(es)g(a)g(hierarc)o(h)o(y)h(of)e(pro)q
(cesses,)i(determining)h(ho)o(w)d(to)h(a)o(v)o(oid)g(this)g(kind)i(of)d
(di\016cult)o(y)j(can)150 2193 y(b)q(ecome)f(problematic.)21
b(T)l(o)13 b(b)q(e)h(precise,)h(on)f(a)f(system)g(with)h(globally-blo)q
(c)o(king)i(sends,)f(the)e(follo)o(wing)150 2250 y(is)k(required)h(in)f
(order)g(to)f(sho)o(w)g(that)g(a)g(co)q(de)h(is)g(deadlo)q(c)o(k)h
(free:)23 b(Let)16 b(eac)o(h)h(pro)q(cess)g(b)q(e)g(a)g(no)q(de)g(in)g
(a)150 2306 y(graph,)i(and)g(let)g(eac)o(h)f(globally-blo)q(c)o(king)k
(send)d(create)f(an)h(arc)f(from)g(the)h(sender)g(to)f(the)h(receiv)o
(er.)150 2363 y(Let)d(the)g(corresp)q(onding)g(receiv)o(e)h(destro)o(y)
e(the)h(arc)f(created)h(b)o(y)g(the)f(send.)23 b(W)l(e)15
b(m)o(ust)g(nev)o(er)h(ac)o(hiev)o(e)150 2419 y(a)i(steady)h(state)f
(\(all)h(sends/receiv)o(es)h(whic)o(h)f(ma)o(y)f(b)q(e)i(reac)o(hed)f
(ha)o(v)o(e)f(b)q(een)i(accoun)o(ted)f(for\))f(where)150
2476 y(there)d(is)h(a)f(cycle.)221 2534 y(F)l(or)e(this)h(reason,)f(it)
h(w)o(as)e(decided)k(that)d(the)h(BLA)o(CS)g(w)o(ould)g(supp)q(ort)g
(lo)q(cally-blo)q(c)o(king)i(p)q(oin)o(t)f(to)150 2591
y(p)q(oin)o(t)k(sends.)29 b(On)19 b(systems)f(nativ)o(ely)h(supp)q
(orting)g(globally-blo)q(c)o(king)i(sends,)e(non-blo)q(c)o(king)h
(sends)150 2647 y(coupled)k(with)e(bu\013ering)h(are)f(used)h(to)f
(create)g(lo)q(cally-blo)q(c)o(kin)q(g)j(sends.)41 b(The)23
b(BLA)o(CS)g(supp)q(ort)150 2704 y(globally-blo)q(c)o(king)18
b(p)q(oin)o(t)e(to)e(p)q(oin)o(t)i(receiv)o(es.)1027
2828 y(40)p eop
%%Page: 41 45
41 44 bop 221 199 a Fs(Up)13 b(till)h(no)o(w,)e(w)o(e)h(ha)o(v)o(e)f
(discussed)i(the)f(blo)q(c)o(king)h(lev)o(els)g(mainly)g(in)g(terms)e
(of)g(p)q(oin)o(t)h(to)g(p)q(oin)o(t)g(mes-)150 256 y(sage)j(passing,)g
(where)g(eac)o(h)h(op)q(eration)f(in)o(v)o(olv)o(es)h(at)e(most)g(t)o
(w)o(o)g(pro)q(cesses.)23 b(The)17 b(concept)f(naturally)150
312 y(extends)g(to)e(the)h(scop)q(ed)h(op)q(erations)g(\(broadcast)e
(and)h(com)o(bine\),)h(ho)o(w)o(ev)o(er.)221 369 y(A)11
b(lo)q(cally-blo)q(c)o(ki)q(ng)k(scop)q(ed)d(op)q(eration)g(w)o(ould)h
(guaran)o(tee)e(to)g(return)h(ev)o(en)g(if)g(no)g(other)g(pro)q(cesses)
150 425 y(in)17 b(the)f(scop)q(e)h(called)h(the)e(routine;)h(a)f
(globally-blo)q(c)o(king)j(scop)q(ed)e(op)q(eration)f(m)o(ust)g(b)q(e)h
(called)h(b)o(y)e(all)150 482 y(pro)q(cesses)g(b)q(efore)g(an)o(y)f
(pro)q(cess)h(is)f(guaran)o(teed)g(to)g(return.)21 b(All)c(scop)q(ed)f
(op)q(erations)g(in)g(the)f(BLA)o(CS)150 538 y(are)g(de\014ned)h(as)f
(globally-blo)q(c)o(king.)221 594 y(The)d(only)h(scop)q(ed)g(op)q
(eration)g(whic)o(h)g Fm(c)n(ould)g Fs(b)q(e)g(programmed)e(as)h
(strictly)h(lo)q(cally-blo)q(c)o(ki)q(ng)i(is)e(the)150
651 y(broadcast/send)g(op)q(eration.)19 b(Ho)o(w)o(ev)o(er,)13
b(since)h(only)g(one)f(pro)q(cess)h(in)g(the)f(scop)q(e)h(w)o(ould)f(b)
q(e)h(calling)h(it,)150 707 y(and)h(the)f(others)g(w)o(ould)h(ha)o(v)o
(e)e(to)h(b)q(e)h(programmed)f(as)g(globally-blo)q(c)o(king,)j(it)d(do)
q(es)h(not)f(add)g(greatly)150 764 y(to)h(the)h(programmabilit)o(y)g
(of)f(the)h(co)q(de)g(to)f(ha)o(v)o(e)g(lo)q(cally-blo)q(c)o(ki)q(ng)j
(broadcast/sends.)24 b(F)l(urther,)17 b(on)150 820 y(some)d(platforms)f
(it)h(is)h(p)q(ossible)g(to)f(ac)o(hiev)o(e)g(considerably)i(b)q(etter)
e(p)q(erformance)g(if)g(broadcast/sends)150 877 y(are)f(allo)o(w)o(ed)h
(to)g(b)q(e)g(globally-blo)q(c)o(king,)i(and)e(w)o(e)g(therefore)f
(de\014ned)i(it)f(to)f(fall)i(in)f(line)i(with)e(the)f(other)150
933 y(scop)q(ed)j(op)q(erations)f(as)g(globally-blo)q(c)o(king.)150
1075 y Fr(C)68 b(BLA)n(CS)23 b(Error)i(Handling)150 1176
y Fs(This)d(section)f(describ)q(es)i(the)e(BLA)o(CS)g(error)g(handling)
h(features.)37 b(The)22 b(BLA)o(CS)f(error)f(handling)150
1233 y(b)q(eha)o(vior)15 b(ma)o(y)e(b)q(e)i(c)o(hanged)f(at)f(compile)j
(time)e(using)h(the)f(C)g(prepro)q(cessor)g(macro)f Fl(BlacsDebugLvl)p
Fs(.)150 1289 y(If)j(y)o(ou)g(are)g(unsure)h(what)e(debug)i(lev)o(el)g
(y)o(our)f(BLA)o(CS)g(are)g(using,)h(this)f(can)g(b)q(e)h(ascertained)g
(b)o(y)f(call)150 1346 y(BLA)o(CS)p 304 1346 14 2 v 17
w(GET)e(\(see)i(Section)g(9.4.1)d(for)i(details\).)221
1402 y(If)f(the)h(BLA)o(CS)g(are)f(compiled)j(with)d(a)h(BLA)o(CS)g
(debug)g(lev)o(el)h(of)e(0,)g(v)o(ery)g(little)i(error)e(c)o(hec)o
(king)h(is)150 1459 y(p)q(erformed.)20 b(A)14 b(few)h(critical)h
(things)e(will)j(b)q(e)e(c)o(hec)o(k)o(ed)g(\(for)e(instance,)i(BLA)o
(CS)p 1543 1459 V 17 w(GRIDINIT)g(will)i(still)150 1515
y(not)e(allo)o(w)h(y)o(ou)f(to)g(allo)q(cate)h(a)f(pro)q(cess)h(grid)g
(with)g(more)f(pro)q(cesses)h(than)f(there)h(are)f(a)o(v)m(ailable\),)i
(but)150 1572 y(for)e(p)q(erformance)g(reasons,)f(the)i(BLA)o(CS)f
(will)i(not)e(c)o(hec)o(k)g(most)g(of)f(the)i(parameters.)221
1628 y(It)21 b(is)h(therefore)f(highly)i(recommended)f(that)f(the)g
(user)h(link)g(his)h(co)q(de)f(to)e(a)h(BLA)o(CS)h(library)150
1684 y(compiled)c(with)f(debug)g(lev)o(el)h(1)e(while)i(debugging)g
(his)f(co)q(de.)24 b(BLA)o(CS)17 b(debug)g(lev)o(el)h(1)e(mainly)i(do)q
(es)150 1741 y(parameter)d(c)o(hec)o(king.)24 b(A)16
b(few)g(other)f(services)i(are)f(also)g(pro)o(vided.)23
b(F)l(or)16 b(instance,)g(the)g(user)h(will)g(b)q(e)150
1797 y(w)o(arned)d(if)h(a)f(pro)q(cess)h(sends)g(a)f(message)g(to)g
(itself.)20 b(Ha)o(ving)15 b(a)f(pro)q(cess)h(send)g(to)f(itself)h(is)g
(legal,)g(but)g(it)150 1854 y(displa)o(ys)j(v)o(ery)e(p)q(o)q(or)g(p)q
(erformance,)h(and)g(requires)g(enough)g(bu\013er)f(space)h(that)f(it)h
(can)f(o)q(ccasionally)150 1910 y(cause)g(hangs)g(for)f(large)h
(messages.)21 b(The)16 b(BLA)o(CS)g(therefore)g(issue)h(a)e(w)o(arning)
h(when)g(this)g(b)q(eha)o(vior)150 1967 y(is)g(detected.)221
2023 y(Man)o(y)f(times)i(the)f(debug)h(lev)o(el)h(0)e(co)q(de)h(will)h
(simply)g(hang,)e(and)g(the)h(dev)o(elop)q(er)g(is)g(left)g(without)150
2080 y(an)o(y)d(clue)i(as)e(to)g(what)f(has)i(gone)f(wrong.)19
b(This)c(ma)o(y)f(b)q(e)h(caused)g(b)o(y)l(,)f(for)g(instance,)h
(trying)f(to)g(receiv)o(e)150 2136 y(from)i(a)h(pro)q(cess)g(whic)o(h)h
(is)f(not)g(in)g(the)g(curren)o(t)g(con)o(text.)25 b(The)17
b(debug)g(lev)o(el)i(1)d(BLA)o(CS)i(can)f(detect)150
2193 y(this)f(sort)e(of)h(a)g(user)g(error,)f(and)h(issue)i(a)d(\(hop)q
(efully)j(helpful\))g(message.)221 2249 y(The)e(BLA)o(CS)h(issue)g
(three)f(t)o(yp)q(es)g(of)g(messages:)206 2344 y(1.)22
b Fm(BLA)o(CS)d(warning)p Fs(:)29 b(BLA)o(CS)21 b(detect)f(risky)g(b)q
(eha)o(vior,)i(but)e(attempt)f(to)h(correct)f(or)h(ignore.)264
2401 y(W)l(arning)15 b(message)g(is)h(prin)o(ted,)f(and)h(execution)g
(pro)q(ceeds.)206 2491 y(2.)22 b Fm(BLA)o(CS)11 b(err)n(or)p
Fs(:)19 b(BLA)o(CS)12 b(detects)g(an)g(error,)g(prin)o(ts)g(an)g(error)
f(message,)h(and)g(kills)i(the)e(mac)o(hine)264 2548
y(via)j(a)g(call)h(to)f(BLA)o(CS)p 669 2548 V 17 w(ABOR)l(T.)206
2638 y(3.)22 b Fm(System)11 b(err)n(or)p Fs(:)18 b(The)11
b(BLA)o(CS)g(receiv)o(e)h(an)f(error)f(message)g(from)g(the)h
(underlying)h(system,)f(whic)o(h)264 2695 y(is)k(then)h(passed)f(on)g
(to)g(the)g(user,)g(and)h(the)f(BLA)o(CS)h(kills)h(the)e(mac)o(hine.)
1027 2828 y(41)p eop
%%Page: 42 46
42 45 bop 150 199 a Fk(C.1)56 b(BLA)n(CS)20 b(W)-5 b(arning)19
b(and)g(Error)f(Messages)150 285 y Fs(All)12 b(BLA)o(CS)g(w)o(arning)f
(messages)g(are)f(prin)o(ted)i(b)o(y)f(the)g(in)o(ternal)h(routine)g
Fl(BlacsWarn)p Fs(,)e(and)h(all)h(BLA)o(CS)150 342 y(error)22
b(messages)g(are)g(prin)o(ted)h(b)o(y)g(the)f(in)o(ternal)i(routine)f
Fl(BlacsErr)p Fs(.)40 b(The)23 b(only)g(real)g(di\013erence)150
398 y(b)q(et)o(w)o(een)12 b Fl(BlacsWarn)d Fs(and)j Fl(BlacsErr)e
Fs(is)i(that)e Fl(BlacsErr)g Fs(calls)i(BLA)o(CS)p 1423
398 14 2 v 17 w(ABOR)l(T)h(after)d(the)h(message)150
454 y(is)16 b(prin)o(ted.)221 511 y(With)c(these)h(cen)o(tral)f
(routines)h(handling)h(BLA)o(CS)e(error)g(messages,)g(it)g(should)i(b)q
(e)f(relativ)o(ely)g(easy)150 567 y(for)k(the)h(programmer)e(to)i(mo)q
(dify)g(error)f(handling)i(if)g(the)f(default)g(routines)g(are)f(not)h
(adequate)g(for)150 624 y(his)c(needs.)20 b(One)13 b(particularly)i
(anno)o(ying)e(problem)h(is)f(that)f(on)h(man)o(y)g(systems)f(a)h(prin)
o(t)g(to)f(the)h(screen)150 680 y(tak)o(es)k(a)g(long)g(time)h(to)f
(\014nish.)28 b Fl(BlacsErr)16 b Fs(ma)o(y)g(then)i(kill)h(the)f(mac)o
(hine)g(b)q(efore)g(the)f(prin)o(t)h(reac)o(hes)150 737
y(the)c(screen,)g(and)f(the)h(error)f(message)g(is)h(lost.)19
b(In)14 b(this)g(case,)g(the)g(user)f(ma)o(y)g(wish)h(to)f(mak)o(e)g
Fl(BlacsErr)150 793 y Fs(w)o(ait)i(b)q(efore)g(killing,)i(or)e(not)g
(kill)i(at)e(all,)g(for)g(instance.)221 850 y(BLA)o(CS)g(w)o(arning)g
(messages)g(ha)o(v)o(e)g(the)g(follo)o(wing)h(form:)150
943 y Fl(BLACS)23 b(WARNING)g('<explanation)f(string>')150
999 y(from)h({<p>,<q>},)g(pnum=<pnum>,)f(Contxt=<ictxt>,)g(on)h(line)h
(<#>)f(of)h(file)f('<fname>'.)221 1093 y Fs(BLA)o(CS)15
b(error)g(messages)f(ha)o(v)o(e)h(the)g(form:)150 1186
y Fl(BLACS)23 b(ERROR)g('<explanation)f(string>')150
1242 y(from)h({<p>,<q>},)g(pnum=<pnum>,)f(Contxt=<ictxt>,)g(on)h(line)h
(<#>)f(of)h(file)f('<fname>'.)221 1335 y Fs(The)15 b(meaning)h(of)f
(these)g(parameters)f(are:)218 1429 y Fn(\017)23 b Fl(explanation)f
(string)11 b Fs(This)h(is)g(the)g(message)f(whic)o(h)h(should)h(help)g
(the)f(user)f(trac)o(k)g(do)o(wn)h(what)264 1485 y(is)h(wrong.)18
b(F)l(or)12 b(example,)h(on)f(an)h(incorrect)f(call)i(to)e(BLA)o(CS)p
1322 1485 V 17 w(GRIDINIT,)h(the)f(user)h(migh)o(t)f(get:)264
1542 y Fl(Process)23 b(0)g(had)h(2)f(x)h(4)g(grid;)f(correct)g(is)h(1)f
(x)h(4)p Fs(.)218 1635 y Fn(\017)f(f)p Fl(p,)g(q)p Fn(g)p
Fs(:)33 b(The)23 b(ro)o(w)e(and)h(column)h(pro)q(cess)f(grid)h(co)q
(ordinates)f(of)g(the)g(pro)q(cess)g(issuing)i(the)264
1692 y(w)o(arning/error.)218 1785 y Fn(\017)f Fl(pnum)p
Fs(:)16 b(This)11 b(will)h(b)q(e)f(the)f(pro)q(cess)g(n)o(um)o(b)q(er)h
(returned)f(in)h(the)g(\014rst)e(argumen)o(t)h(of)f(BLA)o(CS)p
1801 1785 V 17 w(PINF)o(O.)218 1879 y Fn(\017)23 b Fl(ictxt)p
Fs(:)18 b(The)c(in)o(teger)g(con)o(text)f(handle.)21
b(Please)14 b(note)g(that)f(this)h(v)m(alue)h(is)f(not)f(the)h(same)g
(across)264 1935 y(all)19 b(pro)q(cesses.)29 b(F)l(or)18
b(instance,)h(pro)q(cess)g Fn(f)p Fs(0,)f(0)p Fn(g)f
Fs(ma)o(y)h(ha)o(v)o(e)g Fl(ictxt)f Fs(=)i(0)f(and)g(pro)q(cess)h
Fn(f)p Fs(0,)f(1)p Fn(g)264 1992 y Fs(ha)o(v)o(e)13 b
Fl(ictxt)g Fs(=)i(1)e(for)h(the)g(same)g(con)o(text.)k(Ho)o(w)o(ev)o
(er,)13 b(the)h Fl(pnum)g Fs(and)g Fl(ictxt)f Fs(together)g(pro)o(vide)
264 2048 y(an)i(unam)o(biguous)h(pro)q(cess/con)o(text)e(iden)o
(ti\014er.)218 2142 y Fn(\017)23 b Fl(#)p Fs(:)c(The)d(line)h(n)o(um)o
(b)q(er)e(within)i(the)e(\014le)h Fl(fname)f Fs(whic)o(h)h(issued)g
(the)f(w)o(arning.)218 2235 y Fn(\017)23 b Fl(fname)p
Fs(:)c(The)c(\014le)h(name)g(where)f(the)g(routine)h(whic)o(h)g(issued)
g(the)g(w)o(arning/error)e(is)i(lo)q(cated.)221 2328
y(Not)g(all)h(of)g(this)g(information)g(ma)o(y)f(b)q(e)h(a)o(v)m
(ailable)i(at)d(the)g(time)h(an)g(error)f(or)g(w)o(arning)h(is)g
(issued.)150 2385 y(F)l(or)c(instance,)h(if)g(the)g(error)e(o)q(ccurs)i
(b)q(efore)g(the)f(creation)h(of)f(the)h(grid,)f(the)h(pro)q(cess)g
(grid)g(co)q(ordinates)150 2441 y(will)19 b(b)q(e)e(una)o(v)m(ailable.)
28 b(F)l(or)16 b(an)o(y)h(v)m(alue)h(whic)o(h)g(the)f(BLA)o(CS)h
(cannot)f(\014gure)g(out,)g(a)f(-1)h(is)h(prin)o(ted)g(to)150
2498 y(indicate)f(that)d(the)h(v)m(alue)i(is)f(unkno)o(wn.)150
2618 y Fq(C.1.1)51 b(Examples)150 2704 y Fs(A)15 b(few)g(examples)h
(should)g(aid)g(in)g(understanding)h(the)e(BLA)o(CS)h(w)o(arning)f(and)
g(error)g(messages.)1027 2828 y(42)p eop
%%Page: 43 47
43 46 bop 150 199 a Fq(Example)18 b(1)150 287 y Fd(BLACS)j(WARNING)f
('Failure)g(to)i(call)f(BLACS_GET)f(before)g(grid)h(creation)f(makes)h
(code)g(non-portable')150 337 y(from)g({-1,-1},)f(pnum=0,)g(Contxt=-1,)
g(on)i(line)f(-1)g(of)g(file)g('BLACS_GRIDINIT/BLA)o(CS_G)o(RIDMA)o
(P'.)221 432 y Fs(Here)15 b(w)o(e)f(see)h(that)f(the)h(user)g(has)f
(failed)i(to)e(get)h(a)f(system)g(con)o(text)h(via)g(a)f(call)i(to)e
(BLA)o(CS)p 1824 432 14 2 v 17 w(GET,)150 488 y(so)j(that)h(the)f(con)o
(text)h(handle)h(passed)f(to)f(BLA)o(CS)p 1063 488 V
17 w(GRIDINIT)i(or)e(BLA)o(CS)p 1527 488 V 17 w(GRIDMAP)h(is)g(unini-)
150 545 y(tialized.)k(In)15 b(this)g(case,)g(w)o(e)f(w)o(ere)h(able)g
(to)g(detect)f(and)h(correct)g(this,)g(so)f(it)h(is)g(a)g(w)o(arning)g
(and)g(not)f(an)150 601 y(error.)21 b(If)c(b)o(y)f(c)o(hance)g(the)g
(incoming)h(con)o(text)f(had)g(b)q(een)h(set)f(to)f(a)h(v)m(alid)h
(system)f(con)o(text,)f(ho)o(w)o(ev)o(er,)150 658 y(w)o(e)j(w)o(ould)h
(not)e(ha)o(v)o(e)h(b)q(een)i(able)f(to)e(detect)h(the)h(problem,)g
(and)f(the)h(new)f(BLA)o(CS)h(con)o(text)f(w)o(ould)150
714 y(probably)e(b)q(e)g(incorrectly)g(de\014ned.)221
771 y(Since)i(this)g(error)f(o)q(ccurred)h(b)q(efore)f(the)g(pro)q
(cess)h(grid/con)o(text)f(w)o(as)f(created,)i(w)o(e)f(see)g(that)g(the)
150 827 y(pro)q(cess)i(grid)h(co)q(ordinates)g(and)f(the)g(con)o(text)g
(handle)h(are)f(una)o(v)m(ailable,)j(and)e(th)o(us)f(prin)o(ted)h(as)e
(-1.)150 884 y(F)l(urther,)h(at)f(this)h(p)q(oin)o(t)g(the)g(BLA)o(CS)g
(do)f(not)h(kno)o(w)f(whic)o(h)h(\014le)h(the)e(user)h(originally)i
(called)f(from)150 940 y(\(BLA)o(CS)p 322 940 V 17 w(GRIDINIT)g(or)e
(BLA)o(CS)p 788 940 V 17 w(GRIDMAP\))h(and)g(th)o(us)g(are)g(also)h
(unable)g(to)f(supply)h(the)f(line)150 997 y(n)o(um)o(b)q(er.)150
1118 y Fq(Example)f(2)150 1213 y Fl(BLACS)23 b(ERROR)g('CSRC)h(out)f
(of)h(range;)f(CSRC=100,)f(NPCOL=2')150 1269 y(from)h({0,0},)g(pnum=0,)
g(Contxt=0,)g(on)g(line)h(-1)f(of)h(file)f('igerv2d_.c'.)150
1364 y Fs(Here)17 b(w)o(e)g(see)g(that)f(pro)q(cess)i
Fn(f)p Fs(0,)e(0)p Fn(g)g Fs(has)h(issued)h(an)f(illegal)i(receiv)o(e)f
(b)o(y)f(sp)q(ecifying)i(the)e(the)g(pro)q(cess)150 1420
y(column)d(source)g(of)f(the)h(message)f(is)h(100,)f(when)h(there)g
(are)f(in)h(fact)g(only)g(2)f(pro)q(cess)h(columns.)20
b(W)l(e)14 b(see)150 1477 y(that)g(the)i(error)e(is)i(in)g(a)f(call)h
(to)f(IGER)-5 b(V2D,)15 b(and)g(that)f(the)i(line)h(n)o(um)o(b)q(er)e
(w)o(as)f(una)o(v)m(ailable.)150 1600 y Fk(C.2)56 b(System)17
b(Error)h(Messages)150 1686 y Fs(There)c(are)f(times)h(when)g(the)f
(BLA)o(CS)h(will)h(receiv)o(e)g(an)e(error)g(message)g(from)f(the)i
(underlying)h(system)150 1742 y(whic)o(h)i(is)f(not)g(handled)h(b)o(y)f
(the)g(BLA)o(CS.)g(A)o(t)f(this)i(time,)f(the)g(BLA)o(CS)g(will)i(prin)
o(t)e(the)g(system)g(error)150 1799 y(message,)d(and)i(exit.)k(Since)d
(these)e(error)f(messages)h(come)g(from)f(the)h(underlying)i(system,)d
(their)i(form)150 1855 y(will)h(necessarily)f(v)m(ary)f(dep)q(ending)j
(on)d(whic)o(h)g(BLA)o(CS)h(v)o(ersion)f(is)h(b)q(eing)g(used.)20
b(The)15 b(user)f(ma)o(y)f(need)150 1912 y(to)e(obtain)i(a)f(b)q(o)q
(ok)g(describing)i(system)d(error)h(messages)f(to)h(understand)g(the)h
(message.)18 b(F)l(or)11 b(example,)150 1968 y(if)j(the)f(PVM)g(BLA)o
(CS)h(are)e(b)q(eing)j(used,)f(a)f(PVM)f(error)h(n)o(um)o(b)q(er)h
(will)g(b)q(e)g(returned.)20 b(The)13 b(PVM)g(quic)o(k)150
2024 y(reference)i(guide,)g(for)e(example,)i(could)g(then)g(b)q(e)f
(consulted)i(to)d(translate)h(the)g(error)f(n)o(um)o(b)q(er)i(in)o(to)f
(an)150 2081 y(understandable)j(error)d(message.)150
2202 y Fq(C.2.1)51 b(Examples)150 2288 y(Example)18 b(1)150
2383 y Fl(libpvm)23 b([t40025]:)g(pvm_upkint\(\):)f(End)h(of)h(buffer)
150 2440 y(40025:)f(PVM)g(ERROR)h(#-5)f(on)h(call)f(to)h(pvm_upkint)e
(on)i(line)f(25)h(of)f(file)h(iunpack00.c.)150 2534 y
Fs(Here)18 b(w)o(e)g(see)h(that)e(the)h(p)o(vm)g(library)h(has)f(prin)o
(ted)h(out)e(an)i(error)e(message)h(sa)o(ying)g(w)o(e'v)o(e)f(tried)i
(to)150 2591 y(unpac)o(k)d(past)g(the)f(end)i(of)e(our)h(bu\013er.)22
b(The)16 b(BLA)o(CS)g(ha)o(v)o(e)f(passed)h(bac)o(k)g(a)g(PVM)f(error)g
(n)o(um)o(b)q(er)h(of)150 2647 y(-5,)f(whic)o(h)g(is)h(sho)o(wn)f(to)f
(b)q(e)i(`)p Fl(PvmNoData:)22 b(read)h(past)h(end)f(of)h(buffer)p
Fs(')13 b(when)j(lo)q(ok)o(ed)f(up)h(on)f(the)150 2704
y(PVM)i(quic)o(k)g(reference)h(guide.)26 b(F)l(urther,)17
b(w)o(e)g(see)g(that)f(the)h(PVM)g(routine)g(in)h(use)f(when)h(the)f
(error)1027 2828 y(43)p eop
%%Page: 44 48
44 47 bop 150 199 a Fs(o)q(ccurred)16 b(w)o(as)e Fl(pvm_upkint)p
Fs(,)f(and)i(that)f(it)h(w)o(as)f(called)j(on)e(line)h(25)f(of)f(the)h
(BLA)o(CS)h(in)o(ternal)f(routine)150 256 y Fl(iunpack00)p
Fs(.)k(This)c(together)f(with)i(some)f(examination)g(of)g(the)g(co)q
(de)h(in)g(question,)f(rev)o(ealed)h(that)e(in)150 312
y(this)i(case)f(w)o(e)g(w)o(ere)g(trying)g(to)f(receiv)o(e)j(more)d
(data)h(than)g(w)o(as)f(sen)o(t.)221 370 y(This)k(example)g(is)g(esp)q
(ecially)i(w)o(ell)e(c)o(hosen)g(b)q(ecause)g(it)g(illustrates)g(a)f(w)
o(eakness)g(in)h(the)g(presen)o(t)150 427 y(v)o(ersion)e(of)f(the)h
(BLA)o(CS)h(error)e(messages.)21 b(The)16 b(user)g(is)g(informed)h
(that)e(the)h(error)f(o)q(ccurred)h(in)h(the)150 483
y(in)o(ternal)22 b(routine)f Fl(iunpack00)p Fs(,)f(but)h(not)f(what)h
(in)o(terface)g(routine)g(originally)h(called)h Fl(iunpack00)p
Fs(.)150 540 y(Because)17 b(of)e(this)i(lac)o(k)f(in)h(the)f(BLA)o(CS,)
g(the)h(user)f(m)o(ust)f(examine)i(all)g(p)q(ossible)h(candidates)f
(\(in)g(this)150 596 y(case,)c(calls)i(to)e(in)o(teger)g(p)q(oin)o(t)h
(to)f(p)q(oin)o(t)h(receiv)o(es,)g(broadcast/receiv)o(es,)f(or)g(com)o
(bines\),)h(for)f(the)g(error.)150 652 y(This)j(is)f(a)g(problem)h
(that,)e(despite)j(its)e(relativ)o(e)h(simplicit)o(y)l(,)h(has)e(not)g
(y)o(et)g(b)q(een)h(addressed.)150 781 y Fq(Example)i(2)150
883 y Fl(BLACS)23 b(ERROR)g('MPL)h(error)f(PEMPL17)g(on)g(call)h(to)f
(mpc_recv')150 939 y(from)g({0,0},)g(pnum=0,)g(Contxt=0,)g(on)g(line)h
(-1)f(of)h(file)f('Arecv2d00.c'.)221 1039 y Fs(Here)14
b(w)o(e)g(see)h(that)f(the)g(BLA)o(CS)h(are)f(using)h
Fl(BlacsErr)e Fs(to)h(prin)o(t)h(a)f(system)g(error)f(message.)19
b(This)150 1096 y(particular)h(error)f(migh)o(t)g(o)q(ccur)h(when)g
(using)g(the)f(MPL)g(BLA)o(CS.)h(A)o(t)f(this)h(p)q(oin)o(t,)g(the)g
Fm(IBM)f(AIX)150 1152 y(Par)n(al)r(lel)f(Envir)n(onment)g(Instal)r
(lation)f(and)i(Diagnosis)e Fs(man)o(ual)i(should)g(b)q(e)g(consulted.)
29 b(This)19 b(b)q(o)q(ok)150 1209 y(indicates)g(that)e(the)h(error)f
Fl(PEMPL17)g Fs(translates)g(to)g(`The)h(bu\013er)g(sp)q(eci\014ed)i
(for)d(the)h(op)q(eration)g(w)o(as)150 1265 y(to)q(o)d(small)g(to)g
(hold)h(the)f(receiv)o(ed)i(message'.)150 1417 y Fr(D)68
b(Rep)r(eatabilit)n(y)20 b(and)k(coherence)150 1521 y
Fs(Floating)18 b(p)q(oin)o(t)g(computations)f(are)g(not)g(exact)h(on)f
(almost)g(all)i(mo)q(dern)e(arc)o(hitectures.)28 b(This)18
b(lac)o(k)150 1578 y(of)f(precision)i(is)f(particularly)h(problematic)g
(in)f(parallel)h(op)q(erations.)27 b(The)18 b(fact)f(the)h(\015oating)f
(p)q(oin)o(t)150 1634 y(computations)g(are)h(inexact)g(has)f(led)i(us)e
(to)g(classify)i(our)e(algorithms)g(according)h(to)f(whether)h(they)150
1691 y(are)j Fm(r)n(ep)n(e)n(atable)f Fs(and)h(to)f(what)h(degree)g
(they)g(guaran)o(tee)f Fm(c)n(oher)n(enc)n(e)p Fs(.)36
b(A)21 b(routine)g(is)h(rep)q(eatable)f(if)150 1747 y(it)f(is)h(guaran)
o(teed)e(to)g(giv)o(e)i(the)f(same)f(answ)o(er)h(if)g(called)i(m)o
(ultiple)f(times)f(with)h(the)f(same)f(parallel)150 1804
y(con\014guration)13 b(and)h(input.)20 b(A)13 b(routine)h(is)g(coheren)
o(t)f(if)g(all)i(pro)q(cesses)e(selected)i(to)d(receiv)o(e)i(the)g
(answ)o(er)150 1860 y(get)h(iden)o(tical)i(results.)221
1918 y(Please)12 b(note)g(that)g(rep)q(eatabilit)o(y)h(and)f(coherence)
i(do)e(not)f(e\013ect)h(correctness.)19 b(A)12 b(routine)h(ma)o(y)e(b)q
(e)150 1975 y(b)q(oth)g(incoheren)o(t)h(and)g(non-rep)q(eatable,)h(and)
e(still)i(giv)o(e)e(correct)g(output.)18 b(It)11 b(is)h(just)f(that)f
(inaccuracies)150 2031 y(in)17 b(\015oating)f(p)q(oin)o(t)g
(calculations)i(ma)o(y)d(cause)h(the)g(routine)h(to)e(return)h
(di\013ering)h(v)m(alues,)g(all)g(of)e(whic)o(h)150 2088
y(are)g(equally)h(v)m(alid.)221 2146 y(In)h(the)g(follo)o(wing)g
(sections,)g(w)o(e)g(will)h(pro)o(vide)f(illustrativ)o(e)h(examples)g
(as)e(w)o(e)g(examine)i(rep)q(eata-)150 2202 y(bilit)o(y)f(and)e
(coherence)h(in)g(greater)f(detail.)150 2333 y Fk(D.1)56
b(Rep)r(eatabilit)n(y)150 2421 y Fs(Because)17 b(\015oating)g(p)q(oin)o
(t)g(arithmetic)g(is)g(not)f(exact,)g(it)h(is)g(therefore)f(not)g
(truly)h(asso)q(ciativ)o(e)f(\(i.e.,)h(\(a)150 2478 y(+)h(b\))g(+)g(c)g
(ma)o(y)g(not)f(b)q(e)i(the)f(same)f(as)h(a)g(+)g(\(b)g(+)g(c\)\).)28
b(The)18 b(lac)o(k)g(of)f(exact)h(arithmetic)h(can)f(cause)150
2534 y(problems)f(whenev)o(er)f(their)h(exists)f(the)g(p)q(ossibilit)o
(y)j(for)c(reordering)i(of)e(\015oating)h(p)q(oin)o(t)h(calculations.)
150 2591 y(This)i(problem)f(b)q(ecomes)h(prev)m(alen)o(t)g(in)g
(parallel)h(computing)e(due)h(to)e(race)h(conditions)i(in)e(message)150
2647 y(passing.)k(A)16 b(simple)i(example)e(should)h(help)g(in)g
(understanding)g(the)f(problem.)22 b(Let)17 b(us)f(sa)o(y)f(w)o(e)g(ha)
o(v)o(e)150 2704 y(a)i(routine)g(whic)o(h)h(sums)f(n)o(um)o(b)q(ers)g
(stored)f(on)h(di\013eren)o(t)h(pro)q(cesses.)25 b(Let)17
b(us)g(run)h(this)f(routine)g(on)g(4)1027 2828 y(44)p
eop
%%Page: 45 49
45 48 bop 150 199 a Fs(pro)q(cesses,)14 b(with)g(the)g(n)o(um)o(b)q
(ers)g(to)f(b)q(e)h(added)g(b)q(eing)h(the)f(pro)q(cess)g(n)o(um)o(b)q
(ers)g(themselv)o(es.)19 b(Therefore,)150 256 y(pro)q(cess)c(0)g(has)g
(the)h(v)m(alue)g(0)p Fp(:)p Fs(0,)e(pro)q(cess)h(1)g(has)g(the)h(v)m
(alue)g(1)p Fp(:)p Fs(0,)e(etc.)221 312 y(One)21 b(algorithm)h(for)e
(the)h(computation)g(of)g(this)h(result)f(is)h(to)e(ha)o(v)o(e)h(all)h
(pro)q(cesses)g(send)f(their)150 369 y(pro)q(cess)c(n)o(um)o(b)q(ers)h
(to)e(pro)q(cess)h(0;)h(pro)q(cess)f(0)g(then)g(adds)g(them)g(up,)h
(and)f(sends)h(the)f(result)h(bac)o(k)f(to)150 425 y(all)j(pro)q
(cesses.)33 b(So,)19 b(pro)q(cess)h(0)f(w)o(an)o(ts)f(to)h(add)g(a)g(n)
o(um)o(b)q(er)h(to)e(0)p Fp(:)p Fs(0)g(in)i(the)g(\014rst)f(step.)32
b(If)19 b(w)o(e)g(order)150 482 y(our)14 b(receiv)o(es,)h(so)f(that)f
(pro)q(cess)i(0)f(alw)o(a)o(ys)f(receiv)o(es)j(the)e(message)g(from)f
(pro)q(cess)i(1)f(\014rst,)f(then)i(2,)f(and)150 538
y(\014nally)j(3,)d(w)o(e)h(ha)o(v)o(e)g(a)g Fm(r)n(ep)n(e)n(atable)f
Fs(algorithm,)h(the)g(result)h(of)f(whic)o(h)h(is)f(\(\(0)p
Fp(:)p Fs(0)9 b(+)h(1)p Fp(:)p Fs(0\))f(+)i(2)p Fp(:)p
Fs(0\))e(+)h(3)p Fp(:)p Fs(0.)221 595 y(Ho)o(w)o(ev)o(er,)21
b(to)f(get)h(the)g(b)q(est)h(parallel)h(p)q(erformance,)f(it)f(is)h(b)q
(etter)f(not)g(to)g(force)f(a)h(particular)150 651 y(ordering,)16
b(and)g(just)g(ha)o(v)o(e)f(pro)q(cess)i(0)e(add)h(the)g(\014rst)g(a)o
(v)m(ailable)i(n)o(um)o(b)q(er)e(to)f(its)h(v)m(alue,)h(and)f(con)o
(tin)o(ue)150 708 y(to)j(do)h(so)g(un)o(til)h(all)f(n)o(um)o(b)q(ers)g
(ha)o(v)o(e)g(b)q(een)h(added)f(in.)35 b(If)20 b(w)o(e)g(p)q(erform)g
(this)g(optimization,)i(a)d(race)150 764 y(condition)g(o)q(ccurs,)f(b)q
(ecause)h(the)f(order)g(of)f(the)h(op)q(eration)g(is)g(determined)h(b)o
(y)f(the)g(order)f(in)i(whic)o(h)150 820 y(the)h(messages)f(arriv)o(e)h
(on)f(pro)q(cess)h(0,)g(whic)o(h)h(can)f(b)q(e)g(e\013ected)g(b)o(y)g
(an)o(y)f(n)o(um)o(b)q(er)h(of)g(things.)33 b(This)150
877 y(algorithm)15 b(is)h(not)f(rep)q(eatable,)h(b)q(ecause)g(the)g
(answ)o(er)e(ma)o(y)h(v)m(ary)g(b)q(et)o(w)o(een)h(in)o(v)o(o)q
(cations,)f(ev)o(en)h(if)g(the)150 933 y(input)d(is)g(the)g(same.)18
b(F)l(or)12 b(instance,)h(one)g(run)f(migh)o(t)g(pro)q(duce)i(the)e
(sequence)h(\(\(0)p Fp(:)p Fs(0)t(+)t(1)p Fp(:)p Fs(0\))t(+)t(2)p
Fp(:)p Fs(0\))t(+)t(3)p Fp(:)p Fs(0,)150 990 y(while)19
b(a)f(subsequen)o(t)g(run)g(could)h(pro)q(duce)g(\(\(0)p
Fp(:)p Fs(0)10 b(+)i(2)p Fp(:)p Fs(0\))f(+)h(1)p Fp(:)p
Fs(0\))e(+)j(3)p Fp(:)p Fs(0.)26 b(Both)18 b(of)f(these)h(results)g
(are)150 1046 y(correct)c(summations)f(of)h(the)g(giv)o(en)h(n)o(um)o
(b)q(ers,)f(but)g(b)q(ecause)h(of)f(\015oating)g(p)q(oin)o(t)g
(roundo\013,)g(they)g(ma)o(y)150 1103 y(b)q(e)i(di\013eren)o(t.)221
1159 y(Therefore,)f(w)o(e)g(see)h(that)f(a)g(routine)h(is)g
Fm(r)n(ep)n(e)n(atable)f Fs(if)h(m)o(ultiple)h(in)o(v)o(o)q(cations)f
(on)g(the)g(same)f(input)150 1216 y(using)h(the)f(same)g(parallel)i
(setup)e(are)g(guaran)o(teed)g(to)f(pro)q(duce)j(the)e(exact)g(same)g
(result.)221 1272 y(W)l(e)g(classify)i(a)e(routine)h(as)f(not)g(rep)q
(eatable)h(if)g(m)o(ultiple)i(in)o(v)o(o)q(cations)e(on)f(the)h(same)f
(input)h(using)150 1329 y(the)f(same)g(parallel)i(setup)e(ma)o(y)g(pro)
q(duce)h(di\013eren)o(t,)f(but)g(equally)i(v)m(alid)g(results.)150
1451 y Fk(D.2)56 b(Coherence)150 1537 y Fs(W)l(e)13 b(state)e(that)h(a)
g(routine)h(pro)q(duces)h(coheren)o(t)e(output)g(if)h(all)h(pro)q
(cesses)f(are)f(guaran)o(teed)g(to)g(ha)o(v)o(e)g(the)150
1593 y(exact)k(same)g(solution.)23 b(Ob)o(viously)l(,)18
b(almost)e(no)g(algorithm)g(in)o(v)o(olving)h(comm)o(unication)g(is)g
(coheren)o(t)150 1650 y(if)k(comm)o(unication)g(can)f(c)o(hange)g(the)h
(v)m(alues)g(b)q(eing)h(comm)o(unicated.)35 b(Therefore,)21
b(if)g(the)f(parallel)150 1706 y(system)f(b)q(eing)j(studied)f(cannot)e
(guaran)o(tee)h(that)f(comm)o(unication)h(b)q(et)o(w)o(een)h(pro)q
(cesses)f(preserv)o(es)150 1763 y(v)m(alues,)c(none)g(of)e(our)h
(routines)h(are)f(guaran)o(teed)f(to)h(pro)q(duce)h(coheren)o(t)f
(results.)221 1819 y(If)k(comm)o(unication)g(is)h(assumed)e(to)h(b)q(e)
g(coheren)o(t,)g(there)g(are)g(still)h(v)m(arious)f(lev)o(els)i(of)d
(coheren)o(t)150 1876 y(algorithms.)j(Some)15 b(algorithms)g(will)i
(guaran)o(tee)e(coherence)i(only)f(if)f(\015oating)h(p)q(oin)o(t)g(op)q
(erations)f(are)150 1932 y(done)g(the)f(exact)g(same)g(on)g(ev)o(ery)g
(no)q(de.)20 b(W)l(e)14 b(call)i(this)e Fm(homo)n(gene)n(ous)h(c)n
(oher)n(enc)n(e)p Fs(:)k(the)14 b(result)g(will)i(b)q(e)150
1989 y(coheren)o(t)d(of)f(the)h(parallel)h(mac)o(hine)f(is)g
(homogeneous)g(in)g(its)g(handling)h(of)f(\015oating)f(p)q(oin)o(t)h
(op)q(erations.)221 2045 y(A)h(stronger)g(assertion)g(of)g(coherence)h
(is)g Fm(heter)n(o)n(gene)n(ous)g(c)n(oher)n(enc)n(e)p
Fs(,)e(whic)o(h)i(do)q(es)g(not)f(require)h(all)150 2102
y(pro)q(cesses)h(to)e(ha)o(v)o(e)h(the)g(same)g(handling)i(of)e
(\015oating)g(p)q(oin)o(t)h(op)q(erations.)221 2158 y(In)e(general,)h
(a)f(routine)h(that)e(is)i(homogeneous)f(coheren)o(t)g(p)q(erforms)g
(computations)g(redundan)o(tly)150 2215 y(on)i(all)h(no)q(des,)g(so)f
(that)f(all)i(pro)q(cesses)g(get)f(the)g(same)g(answ)o(er)g(only)g(if)h
(all)g(pro)q(cesses)g(p)q(erform)f(arith-)150 2271 y(metic)g(in)g(the)g
(exact)f(same)h(w)o(a)o(y)l(,)e(whereas)i(a)f(routine)h(whic)o(h)g(is)g
(heterogeneous)g(coheren)o(t)g(is)g(usually)150 2328
y(constrained)j(to)e(ha)o(ving)i(one)f(pro)q(cess)g(calculate)i(the)e
(\014nal)h(result,)g(and)f(broadcast)f(it)i(to)e(all)i(other)150
2384 y(pro)q(cesses.)25 b(Let)17 b(us)g(go)f(bac)o(k)g(to)g(our)h
(example)g(of)g(summing)g(the)g(pro)q(cess)g(n)o(um)o(b)q(ers)g(of)f(a)
g(4)h(pro)q(cess)150 2441 y(mac)o(hine)f(for)f(some)f(insigh)o(t)i(in)o
(to)g(these)f(lev)o(els)i(of)d(coherence.)150 2561 y
Fq(D.2.1)52 b(Example)18 b(of)f(Incoherence)150 2647
y Fs(An)h(incoheren)o(t)h(algorithm)e(is)i(one)e(whic)o(h)i(do)q(es)f
(not)f(guaran)o(tee)g(that)g(all)i(pro)q(cesses)f(get)f(the)h(same)150
2704 y(result)f(ev)o(en)f(on)h(a)f(homogeneous)g(system)g(with)h
(coheren)o(t)f(comm)o(unication.)24 b(If)16 b(w)o(e)h(go)e(bac)o(k)i
(to)e(our)1027 2828 y(45)p eop
%%Page: 46 50
46 49 bop 150 199 a Fs(example)16 b(of)f(summing)h(the)g(pro)q(cess)g
(n)o(um)o(b)q(ers,)f(w)o(e)h(can)f(demonstrate)g(this)h(kind)h(of)e(b)q
(eha)o(vior.)21 b(One)150 256 y(w)o(a)o(y)f(to)h(p)q(erform)h(suc)o(h)f
(a)g(sum)h(is)g(to)f(ha)o(v)o(e)g(ev)o(ery)g(pro)q(cess)h(broadcast)f
(its)g(n)o(um)o(b)q(er)h(to)f(all)h(other)150 312 y(pro)q(cesses.)h
(Eac)o(h)16 b(pro)q(cess)h(then)g(adds)f(these)g(n)o(um)o(b)q(ers,)h
(starting)e(with)i(its)f(o)o(wn.)23 b(The)16 b(results)h(eac)o(h)150
369 y(pro)q(cess)e(receiv)o(es)i(w)o(ould)e(then)h(b)q(e:)150
425 y(Pro)q(cess)f(0)g(:)20 b(\(\(0)p Fp(:)p Fs(0)8 b(+)j(1)p
Fp(:)p Fs(0\))e(+)h(2)p Fp(:)p Fs(0\))f(+)h(3)p Fp(:)p
Fs(0)150 482 y(Pro)q(cess)15 b(1)g(:)20 b(\(\(1)p Fp(:)p
Fs(0)8 b(+)j(2)p Fp(:)p Fs(0\))e(+)h(3)p Fp(:)p Fs(0\))f(+)h(0)p
Fp(:)p Fs(0)150 538 y(Pro)q(cess)15 b(2)g(:)20 b(\(\(2)p
Fp(:)p Fs(0)8 b(+)j(3)p Fp(:)p Fs(0\))e(+)h(0)p Fp(:)p
Fs(0\))f(+)h(1)p Fp(:)p Fs(0)150 594 y(Pro)q(cess)15
b(3)g(:)20 b(\(\(3)p Fp(:)p Fs(0)8 b(+)j(0)p Fp(:)p Fs(0\))e(+)h(1)p
Fp(:)p Fs(0\))f(+)h(0)p Fp(:)p Fs(0)221 707 y(All)20
b(of)e(these)h(results)g(are)f(equally)j(v)m(alid,)f(and)f(all)h(ma)o
(y)e(b)q(e)h(di\013eren)o(t)g(from)f(eac)o(h)h(other.)30
b(This)150 764 y(algorithm)16 b(is)h(therefore)f(incoheren)o(t.)24
b(As)16 b(a)g(side)h(note,)f(notice)g(that)g(this)h(algorithm)f(is)g
(rep)q(eatable:)150 820 y(eac)o(h)f(pro)q(cess)h(will)h(get)d(the)i
(same)f(result)g(if)h(the)f(algorithm)g(is)h(called)h(again)e(on)g(the)
g(same)g(data.)150 940 y Fq(D.2.2)52 b(Example)18 b(of)f(Homogeneous)g
(Coherence)150 1026 y Fs(Another)d(w)o(a)o(y)e(to)h(p)q(erform)g(this)h
(summation)g(is)g(to)f(ha)o(v)o(e)g(all)i(pro)q(cesses)f(send)g(their)g
(data)f(to)g(all)h(other)150 1082 y(pro)q(cesses,)g(and)f(to)g(ensure)h
(w)o(e)f(don't)g(ha)o(v)o(e)f(the)i(problem)g(of)f(the)g(previous)h
(example,)g(w)o(e)f(enforce)h(the)150 1139 y(natural)j(ordering.)26
b(Therefore,)17 b(the)h(answ)o(ers)e(eac)o(h)h(no)q(de)h(gets)f(is)g
(\(\(0)p Fp(:)p Fs(0)10 b(+)i(1)p Fp(:)p Fs(0\))e(+)i(2)p
Fp(:)p Fs(0\))e(+)i(3)p Fp(:)p Fs(0.)24 b(This)150 1195
y(answ)o(er)15 b(is)g(the)g(same)g(for)g(all)h(pro)q(cesses)f(only)h
(if)g(all)g(pro)q(cesses)f(do)g(the)h(\015oating)f(p)q(oin)o(t)g
(arithmetic)h(in)150 1252 y(the)f(same)g(w)o(a)o(y)l(.)k(Otherwise,)c
(eac)o(h)g(pro)q(cess)g(ma)o(y)g(mak)o(e)f(di\013eren)o(t)h(\015oating)
g(p)q(oin)o(t)h(errors)e(during)i(the)150 1308 y(addition,)e(leading)g
(to)e(incoherence)j(of)d(the)h(output.)19 b(Notice)13
b(that)f(since)i(w)o(e)e(ha)o(v)o(e)g(forced)h(an)g(ordering)150
1365 y(on)i(the)g(addition,)h(this)g(algorithm)f(is)h(rep)q(eatable.)
150 1485 y Fq(D.2.3)52 b(Example)18 b(of)f(Heterogeneous)g(Coherence)
150 1570 y Fs(In)i(our)e(\014nal)i(example,)g(let)g(us)f(sa)o(y)f(w)o
(e)h(ha)o(v)o(e)f(all)i(pro)q(cesses)g(send)f(the)g(result)h(to)e(pro)q
(cess)h(0,)g(whic)o(h)150 1627 y(adds)d(the)h(n)o(um)o(b)q(ers)f(and)h
(broadcasts)e(the)h(result)h(to)f(the)g(rest)g(of)g(the)g(pro)q
(cesses.)21 b(Since)16 b(one)g(pro)q(cess)150 1683 y(do)q(es)f(all)g
(the)g(computation,)f(it)g(can)h(c)o(ho)q(ose)f(an)o(y)g(order)g(it)h
(wishes)g(and)g(it)f(will)j(giv)o(e)d(coheren)o(t)h(results)150
1740 y(as)e(long)g(as)g(comm)o(unication)g(is)h(itself)g(coheren)o(t.)
19 b(If)13 b(w)o(e)g(do)g(not)g(force)g(a)f(particular)i(order)f(on)g
(the)g(w)o(a)o(y)150 1796 y(w)o(e)j(do)g(the)g(addition,)h(the)f
(algorithm)g(will)h(not)f(b)q(e)h(rep)q(eatable.)23 b(If)16
b(w)o(e)g(force)g(a)f(particular)i(order,)f(it)150 1853
y(will)h(b)q(e)f(rep)q(eatable.)150 1974 y Fk(D.3)56
b(Summing)16 b(it)i(up)150 2060 y Fs(W)l(e)k(ha)o(v)o(e)f(seen)i(that)e
(rep)q(eatabilit)o(y)i(and)f(coherence)h(are)e(separate)h(issues)g
(whic)o(h)h(ma)o(y)e(o)q(ccur)h(in)150 2116 y(parallel)17
b(computations.)j(These)15 b(concepts)h(ma)o(y)e(b)q(e)i(summarized)g
(as:)218 2209 y Fn(\017)23 b Fq(Rep)q(eatabilit)o(y)p
Fs(:)g(The)16 b(routine)g(will)h(yield)h(the)e(exact)f(same)g(result)h
(if)g(it)g(run)g(m)o(ultiple)i(times)264 2266 y(on)c(an)f(iden)o(tical)
j(problem.)k(Eac)o(h)14 b(pro)q(cess)g(ma)o(y)g(get)f(a)h(di\013eren)o
(t)g(result)g(than)g(the)g(others)g(\(i.e.,)264 2322
y(rep)q(eatabilit)o(y)i(do)q(es)f(not)f(imply)i(coherence\),)f(but)g
(that)f(v)m(alue)i(will)g(not)f(c)o(hange)f(if)i(the)e(routine)264
2379 y(is)h(in)o(v)o(ok)o(ed)h(m)o(ultiple)h(times.)218
2472 y Fn(\017)23 b Fq(Homogeneous)c(coherence)p Fs(:)k(All)18
b(pro)q(cesses)g(selected)g(to)e(p)q(ossess)h(the)g(result)g(will)i
(receiv)o(e)264 2529 y(the)c(exact)g(same)g(answ)o(er)f(if:)315
2622 y Fq({)23 b Fs(Comm)o(unication)15 b(do)q(es)h(not)f(c)o(hange)g
(the)g(v)m(alue)h(of)f(the)h(comm)o(unicated)f(data.)315
2695 y Fq({)23 b Fs(All)16 b(pro)q(cesses)g(p)q(erform)f(\015oating)g
(p)q(oin)o(t)h(arithmetic)f(exactly)h(the)f(same.)1027
2828 y(46)p eop
%%Page: 47 51
47 50 bop 218 199 a Fn(\017)23 b Fq(Heterogeneous)i(coherence)p
Fs(:)35 b(All)23 b(pro)q(cesses)g(will)h(receiv)o(e)g(the)e(exact)g
(same)h(answ)o(er)e(if)264 256 y(comm)o(unication)16
b(do)q(es)f(not)g(c)o(hange)g(the)g(v)m(alue)i(of)e(the)g(comm)o
(unicated)h(data.)221 425 y(W)l(e)e(ha)o(v)o(e)g(seen)h(that)e(in)i
(general,)g(lac)o(k)f(of)g(the)g(asso)q(ciativ)o(e)h(prop)q(ert)o(y)f
(for)g(\015oating)g(p)q(oin)o(t)g(calcula-)150 482 y(tions)i(ma)o(y)f
(cause)h(b)q(oth)f(incoherence)j(and/or)d(non-rep)q(eatabilit)o(y)l(.)
23 b(W)l(e)16 b(ha)o(v)o(e)f(seen)h(that)f(algorithms)150
538 y(that)k(rely)i(on)f(redundan)o(t)h(computations)f(are)g(at)g(b)q
(est)g(homogeneous)g(coheren)o(t,)h(and)g(that)e(algo-)150
595 y(rithms)d(in)g(whic)o(h)h(one)f(pro)q(cess)g(broadcasts)f(the)h
(result)g(are)f(heterogeneous)h(coheren)o(t.)21 b(It)16
b(has)g(b)q(een)150 651 y(sho)o(wn)e(that)f(rep)q(eatabilit)o(y)j(do)q
(es)e(not)g(imply)h(coherence,)g(nor)f(do)q(es)g(coherence)h(imply)h
(rep)q(eatabilit)o(y)l(.)221 727 y(Since)j(these)e(issues)i(do)e(not)h
(e\013ect)f(the)h(correctness)f(of)g(the)h(answ)o(er,)f(they)h(ma)o(y)f
(b)q(e)h(ignored)g(in)150 783 y(most)d(cases.)23 b(Ho)o(w)o(ev)o(er,)15
b(in)i(v)o(ery)e(sp)q(eci\014c)j(situations,)f(these)f(issues)h(ma)o(y)
e(b)q(ecome)i(v)o(ery)f(imp)q(ortan)o(t.)150 840 y(One)i(w)o(ould)f
(not)g(w)o(an)o(t)f(to)h(ha)o(v)o(e)g(a)f(stopping)i(criteria)g(based)f
(on)g(incoheren)o(t)h(results,)g(for)f(instance.)150
896 y(A)e(user)g(\014rst)g(writing)g(and)g(debugging)h(a)f(parallel)h
(program)e(ma)o(y)g(wish)i(to)e(enforce)h(rep)q(eatabilit)o(y)h(so)150
953 y(the)f(exact)g(same)g(program)f(sequence)j(o)q(ccurs)e(on)g(ev)o
(ery)g(run,)g(etc.)221 1028 y(In)23 b(the)f(BLA)o(CS,)g(w)o(e)g(sp)q
(eak)h(of)f(coherence)h(and)g(rep)q(eatabilit)o(y)h(only)f(in)g(the)f
(con)o(text)g(of)g(the)150 1084 y(com)o(bine)e(op)q(erations.)31
b(As)19 b(men)o(tioned)h(ab)q(o)o(v)o(e,)f(it)g(is)g(p)q(ossible)i(to)d
(ha)o(v)o(e)h(comm)o(unication)h(whic)o(h)f(is)150 1141
y(incoheren)o(t)e(\(for)f(instance,)h(t)o(w)o(o)e(mac)o(hines)i(whic)o
(h)g(store)f(\015oating)g(p)q(oin)o(t)h(n)o(um)o(b)q(ers)f(di\013eren)o
(tly)i(ma)o(y)150 1197 y(easily)i(pro)q(duce)g(incoheren)o(t)f(comm)o
(unication,)i(since)f(a)e(n)o(um)o(b)q(er)h(stored)g(on)g(mac)o(hine)g
(A)g(ma)o(y)f(not)150 1254 y(ha)o(v)o(e)e(a)g(represen)o(tation)h(on)f
(mac)o(hine)i(B\).)e(Ho)o(w)o(ev)o(er,)f(the)i(BLA)o(CS)g(cannot)f(con)
o(trol)g(this)h(issue.)25 b(W)l(e)150 1310 y(mak)o(e)20
b(the)g(assumption)g(that)f(comm)o(unication)i(is)g(coheren)o(t,)g
(whic)o(h)g(for)e(comm)o(unication)i(implies)150 1367
y(that)14 b(it)i(is)g(also)f(rep)q(eatable.)221 1442
y(F)l(or)k(com)o(bine)j(op)q(erations,)f(the)g(BLA)o(CS)f(allo)o(w)h
(the)g(user)f(to)g(set)g(\015ags)g(indicating)i(he)f(wishes)150
1499 y(com)o(bines)f(to)f(b)q(e)h(rep)q(eatable)h(and/or)e
(heterogeneous)g(coheren)o(t)h(\(see)g(Section)g(9.4)f(for)f(details)j
(on)150 1555 y(setting)15 b(these)h(\015ags\).)221 1630
y(If)k(the)g(BLA)o(CS)h(are)f(instructed)g(to)g(guaran)o(tee)f
(heterogeneous)i(coherency)l(,)h(the)e(BLA)o(CS)g(will)150
1687 y(restrict)c(the)g(top)q(ologies)g(whic)o(h)h(can)f(b)q(e)g(used)h
(so)e(that)g(one)h(pro)q(cess)g(\014gures)g(the)g(\014nal)h(result)f
(of)g(the)150 1743 y(com)o(bine,)g(and)f(if)h(necessary)l(,)f
(broadcasts)f(the)i(answ)o(er)e(to)h(all)h(other)f(pro)q(cesses.)221
1819 y(If)k(the)g(BLA)o(CS)g(are)g(instructed)g(to)f(guaran)o(tee)h
(rep)q(eatabilit)o(y)l(,)i(orderings)e(will)h(b)q(e)g(enforced)f(in)150
1875 y(the)d(top)q(ologies)g(whic)o(h)h(are)f(selected.)23
b(This)17 b(ma)o(y)e(result)i(in)g(loss)f(of)f(p)q(erformance)h(whic)o
(h)h(can)f(range)150 1932 y(from)e(negligable)k(to)c(serious)i(dep)q
(ending)h(on)e(the)h(application.)221 2007 y(A)k(couple)j(of)d
(additional)i(notes)f(are)g(in)g(order.)37 b(W)l(e)21
b(ha)o(v)o(e)f(discussed)j(incoherence)g(and)e(non-)150
2064 y(rep)q(eatabilit)o(y)i(arising)f(as)e(a)h(result)h(of)f
(\015oating)g(p)q(oin)o(t)h(errors.)37 b(This)22 b(migh)o(t)f(lead)h
(the)f(reader)h(to)150 2120 y(susp)q(ect)g(that)e(in)o(teger)h
(calculations)h(are)f(alw)o(a)o(ys)f(rep)q(eatable)i(and)f(coheren)o
(t,)h(since)g(they)f(in)o(v)o(olv)o(e)150 2177 y(exact)12
b(arithmetic.)20 b(This)13 b(is)g(true)f(if)h(o)o(v)o(er\015o)o(w)e(is)
i(ignored.)19 b(With)13 b(o)o(v)o(er\015o)o(w)e(tak)o(en)h(in)o(to)g
(consideration,)150 2233 y(ev)o(en)18 b(in)o(teger)h(calculations)g
(can)f(displa)o(y)i(incoherence)f(and)g(non-rep)q(eatabilit)o(y)l(.)30
b(Therefore,)18 b(if)h(the)150 2289 y(rep)q(eatabilit)o(y)d(or)d
(coherence)i(\015ags)f(are)g(set,)g(the)g(BLA)o(CS)h(will)h(treat)d(in)
o(teger)h(com)o(bines)h(the)f(same)g(as)150 2346 y(\015oating)h(p)q
(oin)o(t)h(com)o(bines)g(in)g(enforcing)g(rep)q(eatabilit)o(y)g(and)g
(coherence)g(guards.)221 2421 y(By)i(their)g(nature,)g(maximization)h
(and)f(minimization)h(should)g(alw)o(a)o(ys)e(b)q(e)i(rep)q(eatable.)28
b(In)19 b(the)150 2478 y(complex)g(precisions,)h(ho)o(w)o(ev)o(er,)e
(the)g(real)g(and)g(imaginary)h(parts)e(m)o(ust)h(b)q(e)h(com)o(bined)g
(in)g(order)f(to)150 2534 y(obtain)d(a)f(magnitude)h(v)m(alue)g(used)g
(to)f(do)g(the)h(comparison)f(\(this)g(is)h(t)o(ypically)h
Fn(j)p Fp(r)q Fn(j)8 b Fs(+)g Fn(j)p Fp(i)p Fn(j)14 b
Fs(or)1749 2496 y Fn(p)p 1787 2496 133 2 v 38 x Fp(r)1809
2521 y Ff(2)1839 2534 y Fs(+)c Fp(i)1900 2521 y Ff(2)1920
2534 y Fs(\).)150 2591 y(This)15 b(allo)o(ws)g(for)g(the)f(p)q
(ossibilit)o(y)j(of)e(heterogeneous)g(incoherence.)21
b(The)15 b(BLA)o(CS)h(therefore)e(restrict)150 2647 y(whic)o(h)g(top)q
(ologies)f(are)g(used)g(for)f(maximization)i(and)f(minimization)i(in)f
(the)f(complex)g(routines)g(when)150 2704 y(the)i(heterogeneous)h
(coherence)g(\015ag)f(is)g(set.)1027 2828 y(47)p eop
%%Page: 48 52
48 51 bop 150 199 a Fr(E)69 b(Broadcast)24 b(T)-6 b(op)r(ologies)150
307 y Fs(This)15 b(app)q(endix)g(discusses)g(the)f(broadcast)g(top)q
(ologies)g(o\013ered)g(b)o(y)g(the)g(presen)o(t)g(BLA)o(CS)g(v)o
(ersions)g(in)150 363 y(greater)g(detail.)221 423 y(Man)o(y)h(factors)h
(e\013ect)g(the)h(c)o(hoice)g(of)g(whic)o(h)g(top)q(ology)f(to)g(use.)
25 b(First,)16 b(the)g(user)h(m)o(ust)f(decide)i(if)150
479 y(an)o(y)f(pro)q(cessor)g(is)h(more)f(imp)q(ortan)o(t)g(than)g
(others.)25 b(F)l(or)17 b(instance,)h(if)g(the)g(source)f(pro)q
(cessor's)g(time)150 536 y(is)h(more)f(imp)q(ortan)o(t)g(than)h(other)f
(pro)q(cessors',)g(a)g(ring)h(top)q(ology)f(is)h(often)f(optimal.)28
b(On)18 b(the)f(other)150 592 y(hand,)e(if)h(ev)o(ery)o(one)f(needs)h
(the)f(information)h(quic)o(kly)l(,)g(some)f(t)o(yp)q(e)g(of)g(tree)g
(is)h(often)f(b)q(est.)221 652 y(Some)10 b(top)q(ologies)i(tie)f(up)h
(the)f(sending)h(pro)q(cessor)e(for)h(large)g(amoun)o(ts)f(of)g(time,)i
(and)f(di\013eren)o(t)g(pro-)150 709 y(cessors)i(get)h(the)f
(information)h(at)f(di\013eren)o(t)h(times)g(dep)q(ending)i(on)e(top)q
(ology)l(.)19 b(Also,)14 b(some)g(top)q(ologies)150 765
y(are)g(\\noisy",)h(i.e.)20 b(man)o(y)14 b(comm)o(unications)h(are)g
(issued)g(sim)o(ultaneously)l(,)h(while)h(others)d(are)g(\\quiet".)150
821 y(Noisy)i(algorithms)g(will)i(cause)e(problems)h(on)f(systems)g
(where)g(net)o(w)o(ork)f(con\015icts)i(are)e(problematic.)150
878 y(Quiet)j(algorithms)e(are)g(lik)o(ely)i(to)e(force)g(some)g(pro)q
(cessors)h(to)e(w)o(ait)h(m)o(uc)o(h)h(longer)g(than)f(they)h(w)o(ould)
150 934 y(if)f(a)f(\\noisy")g(top)q(ology)g(had)g(b)q(een)h(used,)g
(since)g(less)g(comm)o(unication)g(is)f(going)h(on)f(in)h(parallel.)221
994 y(Some)f(top)q(ologies)h(are)f(\\pip)q(elini)q(ng",)j(i.e.,)d(the)h
(\014rst)f(suc)o(h)h(op)q(eration)f(sync)o(hronizes)i(the)e(pro)q(ces-)
150 1050 y(sors)g(so)f(that)h(subsequen)o(t)h(op)q(erations)f(will)i(b)
q(e)f(c)o(heap.)221 1110 y(In)d(the)g(discussion)h(of)e(the)h(presen)o
(tly)h(supp)q(orted)f(top)q(ologies)g(giv)o(en)g(b)q(elo)o(w,)h(w)o(e)e
(use)h(the)g(follo)o(wing)150 1167 y(sym)o(b)q(ols:)26
b Fp(N)384 1174 y Fo(p)404 1167 y Fs(,)19 b(the)f(n)o(um)o(b)q(er)h(of)
f(pro)q(cessors)g(in)o(v)o(olv)o(ed)h(in)g(the)f(op)q(eration,)h(and)g
Fp(T)1618 1174 y Fo(c)1635 1167 y Fs(,)f(the)h(time)g(for)e(a)150
1223 y(complete)f(comm)o(unication)g(\(send)f(and)h(receiv)o(e\).)21
b(Simpli\014ed)d(estimates)d(of)g(the)g(time)h(to)e(p)q(erform)h(a)150
1279 y(giv)o(en)h(algorithm)f(are)g(giv)o(en)g(b)q(elo)o(w.)21
b(F)l(or)14 b(a)h(more)g(complete)h(handling)h(of)e(this)g(topic,)h
(see)f([16)o(])221 1339 y(All)g(\014gures)f(displa)o(ying)i(comm)o
(unication)e(patterns)g(are)f(sho)o(wn)h(with)g Fp(N)1495
1346 y Fo(p)1527 1339 y Fs(=)f(8,)h(b)q(ecause)h(this)f(size)150
1396 y(is)i(adequate)e(to)h(sho)o(w)f(o\013)g(the)i(features)e(of)h
(the)g(top)q(ologies,)g(and)g(is)h(still)g(small)g(enough)f(to)f(\014t)
h(in)o(to)g(a)150 1452 y(reasonable)e(amoun)o(t)f(of)g(space.)19
b(F)l(urther,)13 b(the)f(pro)q(cessors)g(are)h(n)o(um)o(b)q(ered)g
(from)f(0)p Fp(;)c(:)g(:)g(:)t(;)g Fs(\()p Fp(N)1751
1459 y Fo(p)1774 1452 y Fn(\000)d Fs(1\).)19 b(W)l(e)150
1508 y(do)12 b(not)g(sp)q(ecify)i(grid)e(co)q(ordinates)h(b)q(ecause)g
(these)g(broadcasts)e(can)i(op)q(erate)f(on)g(ro)o(ws)f(or)h(columns,)h
(or)150 1565 y(the)f(en)o(tire)h(grid.)19 b(If)12 b(w)o(e)g(instan)o
(tiate)g(suc)o(h)g(a)g(picture)h(as)f(a)f(ro)o(w)g(broadcast,)h(for)f
(instance,)j(these)e(v)m(alues)150 1621 y(are)g(column)g(indices.)21
b(F)l(or)12 b(ease)g(of)f(reference,)i(w)o(e)f(will)h(still)h(refer)d
(to)h(a)f(giv)o(en)i(index)g(as)f(\\pro)q(cessor)f Fp(I)t
Fs(",)150 1678 y(but)i(this)g(should)h(b)q(e)g(tak)o(en)e(to)g(mean)h
(the)g(pro)q(cessor)g(at)f(the)h(I'th)f(p)q(osition)i(in)g(a)e(ro)o(w,)
g(a)h(column,)h(or)e(in)150 1734 y(the)i(grid.)20 b(Please)15
b(note)f(as)f(w)o(ell)i(that)f(the)g(term)f(pro)q(cessor)h(has)g(no)o
(w)g(replaced)h(pro)q(cess.)20 b(W)l(e)14 b(presen)o(t)150
1791 y(timing)19 b(analysis)f(in)h(this)f(section,)h(and)f(they)g(will)
i(not)d(b)q(e)i(accurate)f(if)g(more)f(than)h(one)g(pro)q(cess)g(is)150
1847 y(spa)o(wned)d(to)g(a)g(giv)o(en)g(pro)q(cessor.)221
1907 y(T)l(o)g(b)q(e)h(consisten)o(t,)g(pro)q(cessor)g(0)f(is)h(alw)o
(a)o(ys)f(sho)o(wn)h(as)f(the)h(source)g(\(destination\))g(of)f(the)h
(broad-)150 1963 y(cast)h(\(com)o(bine\).)27 b(Finally)l(,)20
b(a)d(lab)q(el)i Fl(S)24 b(=)g(I)17 b Fs(to)g(the)h(left)g(of)f(a)g
(\014gure)h(indicates)h(that)e(the)h(algorithm)150 2020
y(is)g(in)h(the)e(I'th)h(step.)27 b(F)l(or)18 b(the)f(time)h(analysis)h
(discussed)g(in)g(the)e(text,)h(it)g(is)g(assumed)g(the)f(BLA)o(CS)150
2076 y(are)e(op)q(erating)h(in)g(an)f(en)o(vironmen)o(t)h(where)g(an)f
(arbitrary)g(n)o(um)o(b)q(er)g(of)g(pro)q(cessors)g(ma)o(y)g(b)q(e)h
(comm)o(u-)150 2133 y(nicating)j(sim)o(ultaneously)l(.)28
b(This)18 b(assumption)g(will)h(a\013ect)d(the)i(accuracy)g(of)f(our)g
(prediction)i(if)f(the)150 2189 y(n)o(um)o(b)q(er)e(of)e(actual)i
(links)g(is)g(less)g(than)f(those)g(assumed)g(b)o(y)g(the)h(algorithm.)
221 2249 y(A)o(t)e(the)g(presen)o(t)h(time)g(there)f(are)h(t)o(w)o(o)e
(classes)i(of)f(broadcast)g(top)q(ology)l(.)20 b(The)14
b(\014rst)h(class)f(in)o(v)o(olv)o(es)150 2305 y(top)q(ologies)k(based)
g(on)f(rings.)28 b(The)17 b(second)h(classi\014cation)h(consists)f(of)f
(top)q(ologies)h(based)g(on)g(trees.)150 2362 y(Within)g(these)g
(classes,)g(there)f(are)g(sev)o(eral)g(di\013eren)o(t)h(algorithms.)25
b(F)l(or)17 b(ring)g(top)q(ologies,)h(the)g(main)150
2418 y(di\013erences)i(in)o(v)o(olv)o(e)f(whic)o(h)h(direction)g
(within)g(the)f(ring)g(messages)g(\015o)o(w)f
(\(increasing/decreasing\),)150 2475 y(and)h(the)g(n)o(um)o(b)q(er)g
(of)f(rings)h(the)g(scop)q(e)g(is)h(separated)e(in)o(to)h(\()p
Fp(N)1280 2482 y Fo(r)1298 2475 y Fs(\).)30 b(F)l(or)18
b(tree)h(top)q(ologies,)h(the)f(main)150 2531 y(v)m(ariables)f(in)o(v)o
(olv)o(e)e(the)h(n)o(um)o(b)q(er)f(of)g(branc)o(hes)g(\()p
Fp(N)1036 2538 y Fo(b)1053 2531 y Fs(\))g(at)f(eac)o(h)h(no)q(de)h(of)f
(the)g(tree,)g(and)h(whic)o(h)g(branc)o(h)150 2587 y(is)f(sen)o(t)f(to)
f(\014rst.)221 2647 y(These)h(classes)g(are)f(explained)j(in)e(detail)h
(b)q(elo)o(w,)f(and)g(T)l(able)h(7)e(pro)o(vides)h(a)f(quic)o(k)i
(summation)e(of)150 2704 y(some)i(of)f(the)h(more)f(imp)q(ortan)o(t)h
(prop)q(erties.)23 b(This)16 b(T)l(able)h(sp)q(eci\014es)g(the)f(n)o
(um)o(b)q(er)h(of)e(steps)h(un)o(til)h(the)1027 2828
y(48)p eop
%%Page: 49 53
49 52 bop 150 199 a Fs(algorithm)14 b(completes)h(\(STEPS\),)e(the)i(n)
o(um)o(b)q(er)f(of)g(messages)f(sen)o(t)h(during)h(step)f
Fp(i)g Fs(\(SENDS,)g Fp(S)h Fs(=)e Fp(i)p Fs(\),)150
256 y(the)g(n)o(um)o(b)q(er)g(of)f(pro)q(cessors)h(who)f(are)h
(\014nished)h(with)g(the)e(routine)i(after)e(step)h Fp(i)f
Fs(is)h(complete)h(\(PR)o(OCS)150 312 y(DONE,)j Fp(S)i
Fs(=)d Fp(i)p Fs(\),)h(the)h(time)f(the)h(source)f(pro)q(cessor)h(sp)q
(ends)g(in)g(the)g(algorithm)f(\(SR)o(C)g(TIME\),)g(and)150
369 y(\014nally)22 b(the)g(maxim)o(um)f(time)g(sp)q(en)o(t)g(b)o(y)g
(an)o(y)g(pro)q(cessor)g(in)h(the)f(op)q(eration)g(\(MAX)g(TIME\).)f
(The)150 425 y(analyses)15 b(sho)o(wn)g(in)g(T)l(able)h(7)e(ha)o(v)o(e)
h(b)q(een)h(simpli\014ed)h(b)o(y)e(assuming)g(that)f
Fp(N)1491 432 y Fo(r)1524 425 y Fs(is)i(an)f(ev)o(en)g(m)o(ultiple)h
(of)150 482 y Fp(N)187 489 y Fo(p)206 482 y Fs(,)g(and)f
Fp(N)360 489 y Fo(b)390 482 y Fs(=)f(1,)h(with)h Fp(N)631
489 y Fo(p)666 482 y Fs(an)f(in)o(teger)h(m)o(ultiple)h(of)e(2.)21
b(The)16 b(sp)q(eci\014c)h(top)q(ology)f(section)g(should)g(b)q(e)150
538 y(examined)g(for)f(full)h(details.)p 408 592 1285
2 v 407 649 2 57 v 416 649 V 906 649 V 932 632 a Fp(N)969
639 y Fo(r)988 632 y Fs({RING)p 1385 649 V 280 w(1{TREE)p
1691 649 V 408 651 1285 2 v 408 661 V 407 717 2 57 v
416 717 V 441 700 a(Steps)p 906 717 V 385 w Fp(N)969
707 y Fo(p)989 700 y Fp(=)m(N)1046 707 y Fo(r)p 1385
717 V 1410 700 a Fs(log)1469 711 y Ff(2)1489 700 y Fs(\()p
Fp(N)1544 707 y Fo(p)1563 700 y Fs(\))p 1691 717 V 408
719 1285 2 v 407 775 2 57 v 416 775 V 441 758 a(SENDS,)f
Fp(S)g Fs(=)e Fp(i)p 906 775 V 206 w(N)969 765 y Fo(r)p
1385 775 V 1410 758 a Fs(\(2\))1469 742 y Fo(i)p 1691
775 V 408 777 1285 2 v 407 833 2 57 v 415 833 V 441 816
a Fs(PR)o(OCS)j(DONE,)f Fp(S)g Fs(=)e Fp(i)p 906 833
V 49 w Fs(1)d(+)g Fp(N)1047 823 y Fo(r)1076 816 y Fn(\003)g
Fp(i)p 1385 833 V 285 w Fs(0)p 1691 833 V 408 835 1285
2 v 407 891 2 57 v 416 891 V 441 874 a(SR)o(C)15 b(TIME)p
906 891 V 264 w Fp(N)969 881 y Fo(r)998 874 y Fn(\003)10
b Fp(T)1058 881 y Fo(c)p 1385 891 V 1410 874 a Fs(log)1469
885 y Ff(2)1489 874 y Fs(\()p Fp(N)1544 881 y Fo(p)1563
874 y Fs(\))g Fn(\003)f Fp(T)1650 881 y Fo(c)p 1691 891
V 408 893 1285 2 v 407 961 2 68 v 415 961 V 441 941 a
Fs(MAX)15 b(TIME)p 906 961 V 244 w(\()955 918 y Ff(\()p
Fo(N)997 922 y Fc(p)1009 918 y Fy(\000)-6 b Ff(1\))p
955 930 107 2 v 986 956 a Fo(N)1014 960 y Fc(r)1077 941
y Fs(+)10 b Fp(N)1159 948 y Fo(r)1188 941 y Fn(\000)g
Fs(1\))g Fn(\003)g Fp(T)1344 948 y Fo(c)p 1385 961 2
68 v 1410 941 a Fs(log)1469 952 y Ff(2)1489 941 y Fs(\()p
Fp(N)1544 948 y Fo(p)1563 941 y Fs(\))g Fn(\003)f Fp(T)1650
948 y Fo(c)p 1691 961 V 408 962 1285 2 v 407 1019 2 57
v 415 1019 V 441 1002 a Fs(PIPELINING?)p 906 1019 V 197
w(YES)p 1385 1019 V 388 w(NO)p 1691 1019 V 408 1020 1285
2 v 408 1030 V 662 1108 a(T)l(able)16 b(7:)k(Broadcast)14
b(top)q(ology)h(highligh)o(ts)150 1299 y Fk(E.1)55 b(Broadcast)19
b(Ring)f(T)-5 b(op)r(ologies)150 1388 y Fs(The)21 b(v)m(arious)g(ring)g
(top)q(ologies)g(are)g(discussed)h(b)q(elo)o(w.)37 b(All)22
b(of)e(these)h(top)q(ologies)g(can)g(exp)q(erience)150
1444 y(pip)q(elini)q(ng)e(of)d(v)m(arious)h(degrees.)24
b(Our)17 b(timing)g(mo)q(dels)g(assume)g(that)f(pro)q(cessors)g(are)g
(roughly)h(syn-)150 1501 y(c)o(hronized)i(when)g(en)o(tering)f(the)g
(broadcast.)27 b(Ho)o(w)o(ev)o(er,)17 b(when)i(a)e(ring)i(broadcast)e
(is)h(p)q(erformed,)h(it)150 1557 y(forces)c(an)g(ob)o(vious)h
(ordering)g(on)o(to)e(the)i(pro)q(cessors;)e(i.e,)i(the)f(\014rst)g
(pro)q(cessor)g(in)i(the)e(ring)h(will)h(lea)o(v)o(e)150
1614 y(the)g(op)q(eration)g(b)q(efore)g(the)g(pro)q(cessor)g(whic)o(h)h
(follo)o(ws)f(it)g(in)h(the)f(ring.)25 b(This)18 b(means)e(that)h(once)
g(the)150 1670 y(cost)e(of)f(the)h(\014rst)g(broadcast)f(is)i(paid,)f
(the)g(pro)q(cessors)g(are)g(optimally)h(ordered)f(to)g(p)q(erform)f
(another)150 1727 y(ring)19 b(broadcast.)29 b(The)18
b(time)h(eac)o(h)g(pro)q(cessor)f(incurs)h(for)f(the)h(second)g
(broadcast)e(will)k(b)q(e)e(roughly)150 1783 y Fp(T)177
1790 y Fo(c)194 1783 y Fs(,)g(rather)e(than)i(that)e(giv)o(en)i(in)g
(the)g(text.)29 b(Therefore,)19 b(whenev)o(er)f(a)g(giv)o(en)h(pro)q
(cessor)f(is)h(to)f(issue)150 1840 y(sev)o(eral)f(consecutiv)o(e)i
(broadcasts,)d(use)i(of)e(a)h(ring)h(top)q(ology)f(should)h(b)q(e)g
(considered.)27 b(It)18 b(will)h(result)150 1896 y(in)e(minimization)i
(of)d(the)h(sender's)g(time)g(as)f(usual,)i(but)e(since)i(the)f
(ordering)g(cost)f(is)h(incurred)h(only)150 1953 y(once,)d(it)h(ma)o(y)
e(result)i(in)g(faster)e(o)o(v)o(erall)h(transfer)g(rates)f(as)h(w)o
(ell.)221 2011 y(Pip)q(elines)j(can)e(b)q(e)g(main)o(tained)h(if)f(the)
f(algorithm)h(\015o)o(ws)f(across)g(pro)q(cessors)g(in)i(an)e(orderly)h
(w)o(a)o(y)l(.)150 2067 y(F)l(or)c(example,)i(if)f(the)f(sender)h(of)g
(ro)o(w)e(broadcasts)h(starts)f(out)h(as)h(the)f(\014rst)g(pro)q(cess)h
(column,)h(and)f(then)150 2123 y(is)e(the)g(second,)h(etc,)f(an)g
(increasing)h(ring)f(pip)q(eline)j(will)e(b)q(e)g(main)o(tained.)19
b(If)11 b(the)g(\015o)o(w)f(is)h(in)h(the)e(opp)q(osite)150
2180 y(direction,)17 b(it)f(ma)o(y)f(b)q(e)i(p)q(ossible)g(to)e(set)h
(up)g(a)g(decreasing)g(ring)g(pip)q(elin)q(e.)24 b(The)16
b(e\013ects)g(of)f(pip)q(elini)q(ng)150 2236 y(on)j(broadcast)f(times)h
(will)i(b)q(e)e(discussed)i(in)e(greater)f(detail)i(after)e(all)i
(ring-based)g(top)q(ologies)f(ha)o(v)o(e)150 2293 y(b)q(een)e
(explained.)150 2421 y Fq(Unidirectional)i(Ring)46 b
Fs(Unidirectional)16 b(ring)d(top)q(ologies)g(require)h(the)f(source)g
(pro)q(cessor)g(to)f(issue)150 2478 y(one)j(broadcast,)f(and)h(eac)o(h)
g(pro)q(cessor)g(then)g(receiv)o(es)h(and)f(forw)o(ards)f(the)h
(message.)k(The)c(t)o(w)o(o)f(unidi-)150 2534 y(rectional)f(ring)f(top)
q(ologies)h(are)e(increasing)j(ring)e(\()p Fl(TOP)23
b(=)h('I')p Fs(\),)11 b(and)h(decreasing)h(ring,)f(\()p
Fl(TOP)23 b(=)h('D')p Fs(\).)150 2591 y(These)12 b(algorithms)f(ha)o(v)
o(e)g(the)g(adv)m(an)o(tage)g(that)g(the)g(originating)h(pro)q(cessor)f
(m)o(ust)g(sp)q(end)i(only)e Fp(T)1833 2598 y Fo(c)1862
2591 y Fs(time)150 2647 y(in)18 b(the)e(broadcast.)24
b(Ho)o(w)o(ev)o(er,)16 b(the)h(last)g(pro)q(cessor)f(in)i(the)f(ring)g
(will)h(sp)q(end)g(\()p Fp(N)1584 2654 y Fo(p)1614 2647
y Fn(\000)12 b Fs(1\))e Fn(\003)h Fp(T)1773 2654 y Fo(c)1807
2647 y Fs(time)17 b(in)150 2704 y(algorithm.)36 b(Figures)21
b(3)f(and)h(4)f(resp)q(ectiv)o(ely)i(sho)o(w)e(increasing)i(and)f
(decreasing)g(ring)g(broadcast.)1027 2828 y(49)p eop
%%Page: 50 54
50 53 bop 150 199 a Fs(Unidirectional)16 b(rings)d(are)g(the)g(most)f
(\\quiet")h(algorithms)g(p)q(ossible:)21 b(only)13 b(one)g(pro)q
(cessor)g(is)h(sending)150 256 y(at)h(a)f(time.)253 444
y Fb(\026\025)253 345 y(\027)o(\024)450 444 y(\026\025)450
345 y(\027\024)648 444 y(\026\025)648 345 y(\027)o(\024)845
444 y(\026\025)845 345 y(\027\024)1043 444 y(\026\025)1043
345 y(\027\024)1241 444 y(\026\025)1241 345 y(\027)o(\024)1438
444 y(\026\025)1438 345 y(\027\024)1636 444 y(\026\025)1636
345 y(\027)o(\024)p 353 395 99 4 v 411 393 a Fa(-)p 551
395 V 155 w(-)p 749 395 V 156 w(-)p 946 395 V 156 w(-)p
1144 395 V 155 w(-)p 1341 395 V 156 w(-)p 1539 395 V
155 w(-)293 404 y Fw(0)177 b(1)g(2)f(3)h(4)f(5)h(6)g(7)692
555 y Fs(Figure)15 b(3:)20 b(Increasing)c(ring)g(broadcast)253
803 y Fb(\026\025)253 703 y(\027)o(\024)450 803 y(\026\025)450
703 y(\027\024)648 803 y(\026\025)648 703 y(\027)o(\024)845
803 y(\026\025)845 703 y(\027\024)1043 803 y(\026\025)1043
703 y(\027\024)1241 803 y(\026\025)1241 703 y(\027)o(\024)1438
803 y(\026\025)1438 703 y(\027\024)1636 803 y(\026\025)1636
703 y(\027)o(\024)p 551 753 V 551 751 a Fa(\033)p 749
753 V 156 w(\033)p 946 753 V 155 w(\033)p 1144 753 V
156 w(\033)p 1341 753 V 155 w(\033)p 1539 753 V 156 w(\033)218
753 y Fb(\017)p 218 803 4 19 v 218 838 a(\016)p 253 838
3 4 v 253 753 V 255 838 1497 4 v 1737 751 a Fa(\033)1784
753 y Fb(\014)p 1784 803 4 19 v 1784 838 a(\015)p 1751
838 3 4 v 1751 753 V 293 762 a Fw(0)177 b(1)g(2)f(3)h(4)f(5)h(6)g(7)685
938 y Fs(Figure)16 b(4:)j(Decreasing)d(ring)f(broadcast)150
1122 y Fq(Split)20 b(Ring)47 b Fs(The)16 b(split)i(ring)f(attempts)e
(to)h(alleviate)i(the)e(long)h(w)o(aiting)f(time)h(inheren)o(t)h(in)f
(unidi-)150 1178 y(rectional)d(rings,)g(without)f(t)o(ying)g(up)h(the)f
(originating)h(no)q(de.)20 b(Examining)14 b(Figure)g(5)f(should)h(con)o
(vince)150 1235 y(the)h(reader)g(that)f(the)h(longest)g(time)g(sp)q(en)
o(t)g(in)h(the)f(algorithm)f(is)i(roughly)f Fn(b)p Fp(P)t(=)p
Fs(2)p Fn(c)9 b(\003)g Fp(T)1656 1242 y Fo(c)1673 1235
y Fs(,)15 b(and)g(that)f(the)150 1291 y(source)j(sp)q(ends)h(\(2)11
b Fn(\003)g Fp(T)554 1298 y Fo(c)571 1291 y Fs(\))17
b(time)g(in)h(broadcast.)25 b(The)17 b(split)h(ring)g(top)q(ology)e(is)
i(called)h(b)o(y)e Fl(TOP)23 b(=)h('S')p Fs(.)150 1347
y(Although)13 b(it)g(is)g(unlik)o(ely)i(to)d(b)q(e)h(imp)q(ortan)o(t)f
(in)h(all)h(but)e(the)h(most)f(critical)i(of)e(optimizations,)h(the)g
(user)150 1404 y(should)18 b(kno)o(w)e(that)g(the)h(split)h(ring)f
(sends)h(in)g(the)f(increasing)h(direction)g(\014rst.)24
b(This)18 b(is)f(a)g(relativ)o(ely)150 1460 y(\\quiet")e(algorithm)h
(as)e(only)i(t)o(w)o(o)e(pro)q(cessors)h(will)i(b)q(e)f(sending)g(at)f
(an)o(y)f(one)i(time.)253 1658 y Fb(\026\025)253 1558
y(\027)o(\024)450 1658 y(\026\025)450 1558 y(\027\024)648
1658 y(\026\025)648 1558 y(\027)o(\024)845 1658 y(\026\025)845
1558 y(\027\024)1043 1658 y(\026\025)1043 1558 y(\027\024)1241
1658 y(\026\025)1241 1558 y(\027)o(\024)1438 1658 y(\026\025)1438
1558 y(\027\024)1636 1658 y(\026\025)1636 1558 y(\027)o(\024)p
353 1608 99 4 v 411 1606 a Fa(-)p 551 1608 V 155 w(-)p
749 1608 V 156 w(-)p 946 1608 V 156 w(-)p 1341 1608 V
295 w(\033)p 1539 1608 V 156 w(\033)218 1608 y Fb(\017)p
218 1658 4 19 v 218 1693 a(\016)p 253 1693 3 4 v 253
1608 V 255 1693 1497 4 v 1737 1607 a Fa(\033)1784 1608
y Fb(\014)p 1784 1658 4 19 v 1784 1693 a(\015)p 1751
1693 3 4 v 1751 1608 V 293 1617 a Fw(0)177 b(1)g(2)f(3)h(4)f(5)h(6)g(7)
746 1793 y Fs(Figure)15 b(5:)20 b(Split)c(ring)g(broadcast)150
1969 y Fq(Multiring)47 b Fs(The)20 b(m)o(ultiring)h(algorithm)f(\(also)
g(referred)g(to)f(as)g(m)o(ultipath\))i(pro)o(vides)f(a)g(scalable)150
2025 y(ring)e(algorithm.)28 b(By)18 b(de\014nition,)i(the)d(graph)h
(created)g(b)o(y)f(a)h(m)o(ultiring)h(top)q(ology)e(is)i(not)e(a)g
(ring)i(at)150 2082 y(all,)g(but)g(is)f(instead)h(a)f(sp)q(ecial)h
(kind)g(of)f(tree.)28 b(W)l(e)19 b(call)g(it)f(a)g(ring)g(top)q(ology)g
(despite)h(this,)g(b)q(ecause)150 2138 y(it)f(b)q(eha)o(v)o(es)g(lik)o
(e)h(the)e(true)h(ring)g(top)q(ologies:)25 b(pip)q(elini)q(ng)c(ma)o(y)
c(o)q(ccur,)h(and)g(maxim)o(um)f(time)h(in)h(the)150
2195 y(algorithm)c(scales)h(linearly)h(with)f(the)f(n)o(um)o(b)q(er)g
(of)g(pro)q(cessors)g(in)o(v)o(olv)o(ed.)221 2252 y(In)f(this)g
(algorithm,)g(the)g(user)g(pro)o(vides)g(the)g(n)o(um)o(b)q(er)g(of)g
(rings)g(\()p Fp(N)1389 2259 y Fo(r)1407 2252 y Fs(\))g(up)q(on)g(whic)
o(h)h(the)f(broadcast)150 2308 y(is)22 b(to)f(pro)q(ceed.)39
b(The)22 b(pro)q(cessors)f(participating)h(in)h(the)e(broadcast)g(are)g
(then)h(split)g(up)g(in)o(to)g Fp(N)1932 2315 y Fo(r)150
2365 y Fs(separate)13 b(increasing)h(or)f(decreasing)h(rings)f
(\(increasing)h(rings)g(result)g(if)f(BLA)o(CS)p 1575
2365 14 2 v 17 w(SET)g(is)h(called)h(with)150 2421 y(a)e(p)q(ositiv)o
(e)h Fp(N)389 2428 y Fo(r)407 2421 y Fs(,)f(decreasing)h(rings)f(are)g
(used)h(if)f Fp(N)1012 2428 y Fo(r)1043 2421 y Fs(is)h(set)f(to)f(a)h
(negativ)o(e)g(n)o(um)o(b)q(er\).)19 b(Figure)13 b(6)g(sho)o(ws)f(a)150
2478 y(m)o(ultiring)j(with)f Fp(N)486 2485 y Fo(r)517
2478 y Fs(=)f(3.)19 b(Note)14 b(that)f(the)h(source)g(sends)g(to)f(the)
h(closest)g(ring)h(\014rst,)e(and)h(the)g(farthest)150
2534 y(ring)g(last.)20 b(This)14 b(ma)o(y)f(seem)h(coun)o(ter-pro)q
(ductiv)o(e,)h(in)g(the)f(sense)g(that)f(if)i(w)o(e)e(w)o(ould)h(lik)o
(e)h(to)f(minimize)150 2591 y(link)20 b(con)o(ten)o(tion,)e(sending)i
(the)e(to)g(far)g(ring)g(\014rst)g(mak)o(es)g(more)g(sense.)29
b(Ho)o(w)o(ev)o(er,)18 b(ring)h(top)q(ologies)150 2647
y(are)e(most)f(useful)h(in)h(pip)q(elined)j(co)q(des,)c(where,)g(since)
h(the)f(\015o)o(w)f(of)h(the)g(algorithm)g(pro)q(ceeds)g(in)h(one)150
2704 y(direction)f(across)e(the)h(pro)q(cessors,)f(the)h(time)h(sp)q
(en)o(t)f(b)o(y)f(the)h(nearer)g(pro)q(cessors)g(is)g(more)f(imp)q
(ortan)o(t)1027 2828 y(50)p eop
%%Page: 51 55
51 54 bop 150 199 a Fs(than)15 b(that)g(of)f(the)i(far)e(pro)q
(cessors.)253 538 y Fb(\026\025)253 439 y(\027)o(\024)450
538 y(\026\025)450 439 y(\027\024)648 538 y(\026\025)648
439 y(\027)o(\024)845 538 y(\026\025)845 439 y(\027\024)1043
538 y(\026\025)1043 439 y(\027\024)1241 538 y(\026\025)1241
439 y(\027)o(\024)1438 538 y(\026\025)1438 439 y(\027\024)1636
538 y(\026\025)1636 439 y(\027)o(\024)p 353 489 99 4
v 411 487 a Fa(-)p 551 489 V 155 w(-)p 749 489 V 156
w(-)p 1144 489 V 353 w(-)p 1539 489 V 353 w(-)302 363
y Fb(\033)p 302 427 4 8 v 1070 w(\030)p 1488 427 V 362
363 1070 4 v 302 404 a(\007)p 302 427 4 9 v 758 w(\004)p
1093 427 V 321 404 758 4 v 302 438 4 25 v 1094 438 a
Fa(?)354 b(?)293 498 y Fw(0)177 b(1)g(2)f(3)h(4)f(5)h(6)g(7)624
648 y Fs(Figure)15 b(6:)20 b(Multiring)c(broadcast)f(with)g
Fp(N)1374 655 y Fo(r)1405 648 y Fs(=)e(3)221 788 y(This)g(algorithm)g
(requires)h Fn(d)p Fp(N)753 795 y Fo(p)773 788 y Fp(=)m(N)830
795 y Fo(r)849 788 y Fn(e)f Fs(steps,)g(and)g(at)g(eac)o(h)g(step)g
Fp(N)1378 795 y Fo(r)1410 788 y Fs(sends)g(will)i(b)q(e)f(initiated.)21
b(The)150 845 y(source)12 b(pro)q(cessor)g(is)h(\014nished)h(after)d
(the)i(\014rst)f(step,)g(and)g Fp(N)1188 852 y Fo(r)1219
845 y Fs(pro)q(cessors)g(\014nish)h(eac)o(h)g(step)f(thereafter.)150
901 y(The)f(source)f(pro)q(cessor)g(m)o(ust)g(send)h(to)e(all)j(rings,)
f(and)f(so)g(its)h(time)f(in)i(the)e(algorithm)g(should)i(b)q(e)f
Fp(N)1852 908 y Fo(r)1870 901 y Fn(\003)p Fp(T)1920 908
y Fo(c)1937 901 y Fs(.)150 958 y(The)k(longest)h(time)f(sp)q(en)o(t)h
(in)g(this)f(algorithm)h(will)g(b)q(e)g(roughly)g(\(\()p
Fp(N)1358 965 y Fo(p)1379 958 y Fn(\000)s Fs(1\))p Fp(=)m(N)1515
965 y Fo(r)1543 958 y Fs(+)11 b Fp(N)1626 965 y Fo(r)1654
958 y Fn(\000)g Fs(1\))e Fn(\003)h Fp(T)1810 965 y Fo(c)1827
958 y Fs(.)221 1023 y(Most)15 b(instan)o(tiations)j(of)e(the)h(m)o
(ultiring)h(top)q(ology)f(will)h(b)q(e)g(relativ)o(ely)g(\\quiet",)f
(since)h(at)e(w)o(orst)150 1079 y Fp(N)187 1086 y Fo(r)221
1079 y Fs(pro)q(cessors)f(will)h(b)q(e)g(sending)h(at)d(the)i(same)e
(time.)221 1144 y(Calling)20 b(the)f(m)o(ultiring)h(algorithm)f(is)h
(more)e(complicated)j(than)e(the)g(less)g(general)h(algorithms)150
1200 y(describ)q(ed)h(ab)q(o)o(v)o(e.)32 b(Not)19 b(only)h(m)o(ust)f(a)
g(top)q(ology)g(b)q(e)h(selected,)h(but)f(a)f(n)o(um)o(b)q(er)h(of)f
(rings)g(m)o(ust)g(b)q(e)150 1257 y(passed)c(to)g(the)g(BLA)o(CS.)g
(The)h(general)f(purp)q(ose)h(supp)q(ort)f(routine)h
Fl(BLACS_SET)e Fs(ma)o(y)g(b)q(e)i(used)g(to)e(do)150
1313 y(this.)31 b(Multiring)20 b(is)f(called)h(b)o(y)e(setting)h
Fl(TOP)24 b(=)f('m')p Fs(.)30 b(Here)19 b(is)g(an)g(example)g(of)f(the)
h(recommended)150 1370 y(w)o(a)o(y)14 b(to)h(call)h(the)f(m)o(ultiring)
i(top)q(ology:)269 1491 y Fl(call)24 b(blacs_set\(icontxt,)d(11,)i(3\))
269 1547 y(call)h(dgebs2d\(icontxt,)d('Row',)i('m',)h(m,)f(n,)h(A,)f
(lda\))269 1591 y Fs(.)269 1608 y(.)269 1624 y(.)269
1681 y Fl(call)h(dgebs2d\(icontxt,)d('Column',)i('m',)g(3,)h(2,)f
(work,)h(5\))221 1859 y Fs(Notice)13 b(that)f Fl(BLACS_SET)g
Fs(need)i(only)f(b)q(e)h(called)g(when)g(c)o(hanging)f
Fp(N)1414 1866 y Fo(r)1433 1859 y Fs(,)g(therefore,)f(in)i(the)f
(example)150 1915 y(ab)q(o)o(v)o(e,)21 b(b)q(oth)g(the)g(ro)o(w)f(and)g
(column)i(broadcasts)e(will)i(split)g(their)f(pro)q(cessors)g(in)o(to)f
(3)h(increasing)150 1971 y(rings.)150 2139 y Fq(Pip)q(elining)48
b Fs(All)18 b(ring-based)f(top)q(ologies)g(can)g(displa)o(y)g(pip)q
(elini)q(ng.)27 b(Ho)o(w)o(ev)o(er,)15 b(as)h(the)h(n)o(um)o(b)q(er)f
(of)150 2195 y(rings)e(\()p Fp(N)315 2202 y Fo(r)333
2195 y Fs(\))g(increases,)h(the)f(pip)q(eline)j(adv)m(an)o(tage)c
(tends)h(to)g(decrease.)19 b(After)14 b(a)g(ring)g(broadcast,)f(eac)o
(h)150 2252 y(separate)k(ring)g(is)h(correctly)f(pip)q(elined)k(with)c
(resp)q(ect)h(to)e(the)h(pro)q(cessors)g(within)h(its)g(ring,)f(but)g
(not)150 2308 y(with)c(the)g(source)g(pro)q(cessor.)19
b(As)13 b(the)f(source)h(pro)q(cessor)g(sends)g(more)g(and)g(more)f
(messages,)g(this)i(lac)o(k)150 2365 y(of)i(sync)o(hronization)h(b)q
(ecomes)g(w)o(orse.)23 b(An)16 b(example)h(illustrates)h(this)f
(principle.)26 b(Assume)16 b(w)o(e)g(ha)o(v)o(e)150 2421
y(just)e(\014nished)h(a)f Fp(N)478 2428 y Fo(r)496 2421
y Fs(-ring)g(broadcast.)19 b(A)o(t)13 b(this)i(p)q(oin)o(t)f(the)g
(maxim)o(um)g(cost)f(paid)h(is)h(that)e(giv)o(en)h(in)h(the)150
2478 y(top)q(ology)h(description)j(ab)q(o)o(v)o(e)d(\(call)h(this)h
(time)f Fp(T)1034 2461 y Ff(1)1053 2478 y Fs(\).)24 b(W)l(e)17
b(then)g(rep)q(eat)g(this)g(broadcast)f Fp(k)i Fs(times.)25
b(If)150 2534 y(w)o(e)13 b(ha)o(v)o(e)h(a)f(1-ring,)h(all)h(pro)q
(cessors)e(are)h(sync)o(hronized)h(so)e(that)g(the)h(total)f(cost)g(is)
i(just)e Fp(T)1712 2518 y Ff(1)1739 2534 y Fs(+)7 b Fp(k)i
Fn(\003)e Fp(T)1871 2541 y Fo(c)1887 2534 y Fs(.)20 b(If)150
2591 y Fp(N)187 2598 y Fo(r)218 2591 y Fp(>)13 b Fs(1,)h(ho)o(w)o(ev)o
(er,)f(for)g(eac)o(h)h(iteration)g(b)q(ey)o(ond)h(the)f(\014rst)g(w)o
(e)g(pa)o(y)f(the)h Fp(T)1435 2598 y Fo(c)1466 2591 y
Fs(cost,)g(plus)h(the)f(cost)f(of)h(the)150 2647 y(other)h(sends)h(the)
f(source)h(has)f(had)h(to)e(issue)i(b)q(efore)g(sending)h(to)d(our)i
(ring)f(again.)21 b(Th)o(us,)15 b(in)h(general,)150 2704
y(the)f(cost)g(is)h Fp(T)401 2687 y Ff(1)430 2704 y Fs(+)11
b Fp(k)g Fn(\003)f Fp(N)581 2711 y Fo(r)609 2704 y Fn(\003)g
Fp(T)669 2711 y Fo(c)686 2704 y Fs(.)1027 2828 y(51)p
eop
%%Page: 52 56
52 55 bop 150 199 a Fk(E.2)55 b(Broadcast)19 b(T)-5 b(ree)18
b(T)-5 b(op)r(ologies)150 378 y Fq(Hyp)q(ercub)q(e)46
b Fs(The)18 b(\014rst)g(tree-based)h(top)q(ology)f(is)h(called)h
Fm(hyp)n(er)n(cub)n(e)p Fs(.)29 b(This)19 b(algorithm)f(is)h(a)f(sp)q
(e-)150 434 y(cialized)25 b(broadcast)c(whic)o(h)i(matc)o(hes)f(the)g
(In)o(tel)i(i860's)d(h)o(yp)q(ercub)q(e)j(net)o(w)o(ork.)40
b(It)22 b(uses)h(bit)g(lev)o(el)150 491 y(op)q(erations)c(to)g(ac)o
(hiev)o(e)h(lo)o(w)f(o)o(v)o(erhead)f(in)i(computing)g(source)f(and)h
(destination)g(of)f(messages.)31 b(It)150 547 y(w)o(as)13
b(originally)j(co)q(ded)f(b)o(y)f(Rob)q(ert)h(v)m(an)g(de)f(Geijn[3,)g
(14)o(],)g(and)g(only)h(sligh)o(tly)g(mo)q(di\014ed)h(for)e(inclusion)
150 604 y(in)k(the)g(BLA)o(CS.)g(This)g(top)q(ology)f(requires)i(that)e
Fp(N)1075 611 y Fo(p)1112 604 y Fs(b)q(e)h(an)g(in)o(teger)f(p)q(o)o(w)
o(er)g(of)h(2.)26 b(If)18 b(it)g(is)g(not,)g(the)150
660 y(general)d(tree)g(algorithm)f(describ)q(ed)j(b)q(elo)o(w)e(is)h
(called)g(instead.)k(A)15 b(\014nal)g(detail)h(is)f(that)f(at)h(eac)o
(h)f(no)q(de)150 717 y(in)h(the)f(tree,)f(messages)h(are)f(sen)o(t)h
(to)f(the)h(nearest)g(no)q(de)h(\014rst.)k(This)14 b(broadcast)f
(strategy)g(is)i(sho)o(wn)e(in)150 773 y(Figure)i(7.)221
877 y(Hyp)q(ercub)q(e)e(broadcasts)e(are)g(most)g(useful)i(when)f
(getting)g(the)f(information)h(out)g(to)f(all)h(pro)q(cessors)150
934 y(is)20 b(more)g(imp)q(ortan)o(t)f(than)h(sa)o(ving)f(origin)i(no)q
(de)f(time.)34 b(It)20 b(requires)h(all)f(no)q(des)h(to)e(sp)q(end)i
(roughly)150 990 y Fp(T)177 997 y Fo(c)202 990 y Fn(\003)9
b Fp(l)q(og)293 997 y Ff(2)312 990 y Fs(\()p Fp(N)367
997 y Fo(p)386 990 y Fs(\))14 b(time)g(in)i(the)e(broadcast.)19
b(Hyp)q(ercub)q(e)d(broadcasts)e(are)g(relativ)o(ely)h(\\noisy",)f
(since)i(the)150 1047 y(n)o(um)o(b)q(er)f(of)e(pro)q(cessors)h(sending)
i(at)e(one)g(time)h(gro)o(ws)e(with)h Fp(N)1251 1054
y Fo(p)1271 1047 y Fs(.)19 b(In)c(the)f(last)h(step)f(of)g(the)g
(broadcast,)150 1103 y Fp(N)187 1110 y Fo(p)206 1103
y Fp(=)p Fs(2)h(pro)q(cessors)g(will)i(b)q(e)f(sending)g(sim)o
(ultaneously)l(.)150 1496 y Fq(General)d(T)l(ree)45 b
Fs(The)12 b(\014nal)g(top)q(ology)f(that)f(is)i(supp)q(orted)g(is)g
(the)f(general)h(tree)f(broadcast.)18 b(It)11 b(allo)o(ws)150
1553 y(the)k(user)f(to)g(c)o(ho)q(ose)h(the)f(n)o(um)o(b)q(er)h(of)f
(branc)o(hes)h(\()p Fp(N)1058 1560 y Fo(b)1074 1553 y
Fs(\))g(at)e(eac)o(h)i(step)g(in)g(the)g(broadcast)e(tree.)20
b(Figures)150 1609 y(8,)f(9)g(and)g(10)f(sho)o(w)h(general)g(tree)g
(broadcasts)f(with)h Fp(N)1148 1616 y Fo(b)1184 1609
y Fs(=)g(1)p Fp(;)8 b Fs(2)p Fp(;)g Fs(3.)29 b(Note)18
b(that)g(general)i(tree)f(with)150 1666 y Fp(N)187 1673
y Fo(b)216 1666 y Fs(=)13 b(1)g(is)i(a)e(h)o(yp)q(ercub)q(e)i
(broadcast)e(where)g(at)g(eac)o(h)h(no)q(de)g(in)g(the)g(tree,)f(the)h
(no)q(de)g(furthest)f(from)g(the)150 1722 y(presen)o(t)j(no)q(de)h(is)f
(sen)o(t)g(to)g(\014rst.)22 b(This)17 b(tends)f(to)g(minimize)i(link)f
(con)o(ten)o(tions,)f(if)h(the)f(assumption)g(is)150
1779 y(made)f(that)g(pro)q(cessors)g(far)f(a)o(w)o(a)o(y)g(from)g(eac)o
(h)i(other)e(tend)i(not)f(to)f(share)h(the)h(same)e(link.)221
1883 y(With)e(this)g(algorithm,)g Fp(N)674 1890 y Fo(p)706
1883 y Fs(do)q(es)g Fm(not)g Fs(ha)o(v)o(e)f(to)g(b)q(e)i(an)f(in)o
(teger)g(p)q(o)o(w)o(er)f(of)h Fp(N)1515 1890 y Fo(b)1532
1883 y Fs(.)18 b(The)13 b(timing)f(analysis)150 1939
y(for)18 b(this)i(algorithm)f(is)g(relativ)o(ely)h(complex,)g(so)f(w)o
(e)g(do)f(not)h(repro)q(duce)h(it)f(here)g(\(analysis)h(for)e(the)150
1996 y(most)c(common)h(use,)g Fp(N)571 2003 y Fo(b)601
1996 y Fs(=)e(1)i(is)g(sho)o(wn)g(in)h(T)l(able)g(7\).)j(See)d([16)o(])
f(for)g(full)h(details.)221 2100 y(General)d(tree)h(broadcasts)e(are)h
(ob)o(viously)i(\\noisy",)e(and)g(the)h(greater)e Fp(N)1501
2107 y Fo(b)1532 2100 y Fs(and)h Fp(N)1655 2107 y Fo(p)1688
2100 y Fs(are,)g(the)g(more)150 2156 y(\\noisy")k(the)h(algorithm)f(b)q
(ecomes.)27 b(This)18 b(top)q(ology)f(ma)o(y)f(b)q(e)i(called)h(in)f
(sev)o(eral)g(w)o(a)o(ys.)25 b(If)17 b(the)h(user)150
2213 y(sets)g Fl(TOP)24 b(=)f('t')p Fs(,)c(the)g(routine)g
Fl(BLACS_SET)e Fs(should)j(b)q(e)f(used)g(in)h(the)e(same)h(w)o(a)o(y)e
(as)h(discussed)j(for)150 2269 y(m)o(ultiring.)g(An)16
b(example)g(should)g(clarify)g(this:)269 2374 y Fl(call)24
b(blacs_set\(icontxt,)d(11,)i(2\))269 2430 y(call)h(dgebs2d\(icontxt,)d
('Row',)i('t',)h(m,)f(n,)h(A,)f(lda\))221 2591 y Fs(This)17
b(w)o(ould)h(call)h(the)e(general)h(tree)f(algorithm)g(with)h
Fp(N)1231 2598 y Fo(b)1264 2591 y Fs(=)e(2.)26 b(The)18
b(w)o(a)o(ys)e(to)h(call)h(the)f(general)150 2647 y(tree)e(broadcast)f
(are)h(summarized)h(b)q(elo)o(w.)1027 2828 y(52)p eop
%%Page: 53 57
53 56 bop 461 280 a Fb(\032\031)461 164 y(\033\030)461
579 y(\032\031)461 462 y(\033\030)631 579 y(\032\031)631
462 y(\033\030)461 878 y(\032\031)461 761 y(\033\030)631
878 y(\032\031)631 761 y(\033\030)801 878 y(\032\031)801
761 y(\033)q(\030)972 878 y(\032\031)972 761 y(\033\030)461
1176 y(\032\031)461 1060 y(\033\030)631 1176 y(\032\031)631
1060 y(\033\030)801 1176 y(\032\031)801 1060 y(\033)q(\030)972
1176 y(\032\031)972 1060 y(\033\030)1142 1176 y(\032\031)1142
1060 y(\033\030)1312 1176 y(\032\031)1312 1060 y(\033\030)1482
1176 y(\032\031)1482 1060 y(\033\030)1652 1176 y(\032\031)1652
1060 y(\033)q(\030)238 232 y Fp(S)16 b Fs(=)d(0)238 531
y Fp(S)j Fs(=)d(1)238 830 y Fp(S)j Fs(=)d(2)238 1129
y Fp(S)j Fs(=)d(3)155 b(0)508 830 y(0)508 531 y(0)508
232 y(0)678 1129 y(1)678 830 y(1)678 531 y(1)849 1129
y(2)849 830 y(2)1019 1129 y(3)1019 830 y(3)1189 1129
y(4)147 b(5)g(6)h(7)521 320 y Fa(J)548 361 y(J)576 403
y(J)604 444 y(J)622 471 y(J)-28 b(^)521 608 y(Z)562 639
y(Z)604 670 y(Z)645 702 y(Z)687 733 y(Z)728 764 y(Z)766
792 y(Z)-42 b(~)691 608 y(Z)732 639 y(Z)774 670 y(Z)815
702 y(Z)857 733 y(Z)898 764 y(Z)936 792 y(Z)g(~)521 890
y(P)562 904 y(P)604 917 y(P)645 931 y(P)687 945 y(P)728
959 y(P)770 973 y(P)811 987 y(P)853 1000 y(P)894 1014
y(P)936 1028 y(P)977 1042 y(P)1019 1056 y(P)1060 1070
y(P)1102 1083 y(P)1106 1085 y(P)g(q)691 890 y(P)732 904
y(P)774 917 y(P)815 931 y(P)857 945 y(P)898 959 y(P)940
973 y(P)982 987 y(P)1023 1000 y(P)1065 1014 y(P)1106
1028 y(P)1148 1042 y(P)1189 1056 y(P)1231 1070 y(P)1272
1083 y(P)1276 1085 y(P)g(q)861 890 y(P)903 904 y(P)944
917 y(P)986 931 y(P)1027 945 y(P)1069 959 y(P)1110 973
y(P)1152 987 y(P)1193 1000 y(P)1235 1014 y(P)1276 1028
y(P)1318 1042 y(P)1359 1056 y(P)1401 1070 y(P)1442 1083
y(P)1446 1085 y(P)g(q)1031 890 y(P)1073 904 y(P)1114
917 y(P)1156 931 y(P)1197 945 y(P)1239 959 y(P)1280 973
y(P)1322 987 y(P)1363 1000 y(P)1405 1014 y(P)1446 1028
y(P)1488 1042 y(P)1529 1056 y(P)1571 1070 y(P)1612 1083
y(P)1617 1085 y(P)g(q)546 1289 y Fs(Figure)16 b(7:)j(Hyp)q(ercub)q(e)e
(broadcast,)d(nearest)h(no)q(de)h(\014rst.)p 413 1383
1274 2 v 412 1439 2 57 v 421 1439 V 447 1422 a(TOP)p
570 1439 V 50 w(Explanation)p 1677 1439 V 1686 1439 V
413 1441 1274 2 v 413 1451 V 412 1507 2 57 v 421 1507
V 447 1490 a('1')p 570 1507 V 100 w(tree)f(with)g Fp(N)827
1497 y Fo(b)856 1490 y Fs(=)e(1.)p 1677 1507 V 1686 1507
V 413 1509 1274 2 v 412 1566 2 57 v 421 1566 V 447 1549
a('2')p 570 1566 V 100 w(tree)i(with)g Fp(N)827 1556
y Fo(b)856 1549 y Fs(=)e(2.)p 1677 1566 V 1686 1566 V
413 1567 1274 2 v 412 1624 2 57 v 421 1624 V 447 1607
a('3')p 570 1624 V 100 w(tree)i(with)g Fp(N)827 1614
y Fo(b)856 1607 y Fs(=)e(3.)p 1677 1624 V 1686 1624 V
413 1625 1274 2 v 412 1682 2 57 v 421 1682 V 447 1665
a('4')p 570 1682 V 100 w(tree)i(with)g Fp(N)827 1672
y Fo(b)856 1665 y Fs(=)e(4.)p 1677 1682 V 1686 1682 V
413 1683 1274 2 v 412 1740 2 57 v 421 1740 V 447 1723
a('5')p 570 1740 V 100 w(tree)i(with)g Fp(N)827 1730
y Fo(b)856 1723 y Fs(=)e(5.)p 1677 1740 V 1686 1740 V
413 1742 1274 2 v 412 1798 2 57 v 421 1798 V 447 1781
a('6')p 570 1798 V 100 w(tree)i(with)g Fp(N)827 1788
y Fo(b)856 1781 y Fs(=)e(6.)p 1677 1798 V 1686 1798 V
413 1800 1274 2 v 412 1856 2 57 v 421 1856 V 447 1839
a('7')p 570 1856 V 100 w(tree)i(with)g Fp(N)827 1846
y Fo(b)856 1839 y Fs(=)e(7.)p 1677 1856 V 1686 1856 V
413 1858 1274 2 v 412 1914 2 57 v 421 1914 V 447 1897
a('8')p 570 1914 V 100 w(tree)i(with)g Fp(N)827 1904
y Fo(b)856 1897 y Fs(=)e(8.)p 1677 1914 V 1686 1914 V
413 1916 1274 2 v 412 1972 2 57 v 421 1972 V 447 1955
a('9')p 570 1972 V 100 w(tree)i(with)g Fp(N)827 1962
y Fo(b)856 1955 y Fs(=)e(9.)p 1677 1972 V 1686 1972 V
413 1974 1274 2 v 412 2030 2 57 v 421 2030 V 447 2013
a('t')p 570 2030 V 105 w(tree)i(with)g Fp(N)827 2020
y Fo(b)856 2013 y Fs(=)h Fl(I)p Fs(,where)f Fl(I)g Fs(is)h(set)f(b)o(y)
g(call)h(to)f Fl(BLACS)p 1556 2013 15 2 v 16 w(SET)p
Fs(.)p 1677 2030 2 57 v 1686 2030 V 413 2032 1274 2 v
412 2089 2 57 v 421 2089 V 447 2072 a('f)t(')p 570 2089
V 105 w(p)q(erform)g(fully-connected)i(broadcast,)d(i.e.)21
b Fp(N)1419 2079 y Fo(b)1448 2072 y Fs(=)13 b Fp(N)5
b(p)k Fn(\000)i Fs(1)p 1677 2089 V 1686 2089 V 413 2090
1274 2 v 413 2100 V 1027 2828 a(53)p eop
%%Page: 54 58
54 57 bop 414 409 a Fb(\032\031)414 293 y(\033\030)414
708 y(\032\031)414 592 y(\033\030)1095 708 y(\032\031)1095
592 y(\033\030)414 1007 y(\032\031)414 891 y(\033\030)754
1007 y(\032\031)754 891 y(\033)q(\030)1095 1007 y(\032\031)1095
891 y(\033\030)1435 1007 y(\032\031)1435 891 y(\033\030)414
1306 y(\032\031)414 1190 y(\033\030)584 1306 y(\032\031)584
1190 y(\033\030)754 1306 y(\032\031)754 1190 y(\033)q(\030)925
1306 y(\032\031)925 1190 y(\033\030)1095 1306 y(\032\031)1095
1190 y(\033\030)1265 1306 y(\032\031)1265 1190 y(\033\030)1435
1306 y(\032\031)1435 1190 y(\033\030)1605 1306 y(\032\031)1605
1190 y(\033)q(\030)192 362 y Fp(S)15 b Fs(=)e(0)192 661
y Fp(S)i Fs(=)e(1)192 960 y Fp(S)i Fs(=)e(2)192 1259
y Fp(S)i Fs(=)e(3)155 b(0)461 960 y(0)461 661 y(0)461
362 y(0)632 1259 y(1)147 b(2)802 960 y(2)972 1259 y(3)g(4)1142
960 y(4)1142 661 y(4)1312 1259 y(5)h(6)1483 960 y(6)1653
1259 y(7)474 422 y Fa(P)515 435 y(P)557 449 y(P)598 463
y(P)640 477 y(P)681 491 y(P)723 505 y(P)764 518 y(P)806
532 y(P)847 546 y(P)889 560 y(P)930 574 y(P)972 588 y(P)1013
601 y(P)1055 615 y(P)1059 617 y(P)-42 b(q)474 738 y(Z)515
769 y(Z)557 800 y(Z)598 831 y(Z)640 862 y(Z)681 893 y(Z)719
921 y(Z)g(~)1155 738 y(Z)1196 769 y(Z)1238 800 y(Z)1279
831 y(Z)1321 862 y(Z)1362 893 y(Z)1399 921 y(Z)g(~)474
1047 y(J)501 1088 y(J)529 1130 y(J)557 1171 y(J)575 1198
y(J)-28 b(^)814 1047 y(J)842 1088 y(J)870 1130 y(J)897
1171 y(J)915 1198 y(J)g(^)1155 1047 y(J)1182 1088 y(J)1210
1130 y(J)1238 1171 y(J)1256 1198 y(J)g(^)1495 1047 y(J)1523
1088 y(J)1550 1130 y(J)1578 1171 y(J)1596 1198 y(J)g(^)589
1419 y Fs(Figure)15 b(8:)20 b(General)15 b(tree)g(broadcast)g(with)g
Fp(N)1396 1426 y Fo(b)1426 1419 y Fs(=)e(1)414 1846 y
Fb(\032\031)414 1730 y(\033\030)414 2153 y(\032\031)414
2037 y(\033\030)925 2153 y(\032\031)925 2037 y(\033\030)1435
2153 y(\032\031)1435 2037 y(\033\030)414 2452 y(\032\031)414
2336 y(\033\030)584 2452 y(\032\031)584 2336 y(\033\030)754
2452 y(\032\031)754 2336 y(\033)q(\030)925 2452 y(\032\031)925
2336 y(\033\030)1095 2452 y(\032\031)1095 2336 y(\033\030)1265
2452 y(\032\031)1265 2336 y(\033\030)1435 2452 y(\032\031)1435
2336 y(\033\030)1605 2452 y(\032\031)1605 2336 y(\033)q(\030)192
1807 y Fp(S)i Fs(=)e(0)192 2106 y Fp(S)i Fs(=)e(1)192
2405 y Fp(S)i Fs(=)e(2)155 b(0)461 2106 y(0)461 1807
y(0)632 2405 y(1)147 b(2)g(3)972 2106 y(3)1142 2405 y(4)g(5)h(6)1483
2106 y(6)1653 2405 y(7)474 1855 y Fa(X)515 1865 y(X)557
1876 y(X)598 1886 y(X)640 1896 y(X)681 1907 y(X)723 1917
y(X)764 1928 y(X)806 1938 y(X)847 1948 y(X)889 1959 y(X)930
1969 y(X)972 1979 y(X)1013 1990 y(X)1055 2000 y(X)1096
2011 y(X)1138 2021 y(X)1179 2031 y(X)1221 2042 y(X)1262
2052 y(X)1304 2063 y(X)1346 2073 y(X)1387 2083 y(X)1391
2084 y(X)-42 b(z)474 1865 y(H)515 1886 y(H)557 1907 y(H)598
1928 y(H)640 1948 y(H)681 1969 y(H)723 1990 y(H)764 2011
y(H)806 2031 y(H)847 2052 y(H)885 2071 y(H)g(j)474 2183
y(Z)515 2214 y(Z)557 2245 y(Z)598 2276 y(Z)640 2307 y(Z)681
2339 y(Z)719 2367 y(Z)g(~)984 2183 y(Z)1026 2214 y(Z)1067
2245 y(Z)1109 2276 y(Z)1150 2307 y(Z)1192 2339 y(Z)1229
2367 y(Z)g(~)474 2193 y(J)501 2235 y(J)529 2276 y(J)557
2318 y(J)575 2345 y(J)-28 b(^)984 2193 y(J)1012 2235
y(J)1040 2276 y(J)1067 2318 y(J)1085 2345 y(J)g(^)1495
2193 y(J)1523 2235 y(J)1550 2276 y(J)1578 2318 y(J)1596
2345 y(J)g(^)596 2565 y Fs(Figure)16 b(9:)j(General)d(tree)f(broadcast)
f(with)i Fp(N)1404 2572 y Fo(b)1433 2565 y Fs(=)d(2)1027
2828 y(54)p eop
%%Page: 55 59
55 58 bop 414 1128 a Fb(\032\031)414 1012 y(\033\030)414
1435 y(\032\031)414 1319 y(\033\030)1095 1435 y(\032\031)1095
1319 y(\033\030)414 1734 y(\032\031)414 1618 y(\033\030)584
1734 y(\032\031)584 1618 y(\033\030)754 1734 y(\032\031)754
1618 y(\033)q(\030)925 1734 y(\032\031)925 1618 y(\033\030)1095
1734 y(\032\031)1095 1618 y(\033\030)1265 1734 y(\032\031)1265
1618 y(\033\030)1435 1734 y(\032\031)1435 1618 y(\033\030)1605
1734 y(\032\031)1605 1618 y(\033)q(\030)192 1089 y Fp(S)15
b Fs(=)e(0)192 1388 y Fp(S)i Fs(=)e(1)192 1687 y Fp(S)i
Fs(=)e(2)155 b(0)461 1388 y(0)461 1089 y(0)632 1687 y(1)147
b(2)g(3)g(4)1142 1388 y(4)1312 1687 y(5)h(6)f(7)474 1140
y Fa(P)515 1154 y(P)557 1168 y(P)598 1181 y(P)640 1195
y(P)681 1209 y(P)723 1223 y(P)764 1237 y(P)806 1251 y(P)847
1264 y(P)889 1278 y(P)930 1292 y(P)972 1306 y(P)1013
1320 y(P)1055 1334 y(P)1059 1335 y(P)-42 b(q)474 1464
y(Z)515 1496 y(Z)557 1527 y(Z)598 1558 y(Z)640 1589 y(Z)681
1620 y(Z)719 1648 y(Z)g(~)1155 1464 y(Z)1196 1496 y(Z)1238
1527 y(Z)1279 1558 y(Z)1321 1589 y(Z)1362 1620 y(Z)1399
1648 y(Z)g(~)474 1454 y(H)515 1475 y(H)557 1496 y(H)598
1516 y(H)640 1537 y(H)681 1558 y(H)723 1579 y(H)764 1599
y(H)806 1620 y(H)847 1641 y(H)885 1660 y(H)g(j)1155 1454
y(H)1196 1475 y(H)1238 1496 y(H)1279 1516 y(H)1321 1537
y(H)1362 1558 y(H)1404 1579 y(H)1445 1599 y(H)1487 1620
y(H)1528 1641 y(H)1566 1660 y(H)g(j)474 1475 y(J)501
1516 y(J)529 1558 y(J)557 1599 y(J)575 1626 y(J)-28 b(^)1155
1475 y(J)1182 1516 y(J)1210 1558 y(J)1238 1599 y(J)1256
1626 y(J)g(^)585 1847 y Fs(Figure)15 b(10:)20 b(General)15
b(tree)g(broadcast)g(with)g Fp(N)1415 1854 y Fo(b)1444
1847 y Fs(=)e(3)1027 2828 y(55)p eop
%%Page: 56 60
56 59 bop 150 199 a Fr(F)69 b(Com)n(bine)21 b(T)-6 b(op)r(ologies)150
302 y Fs(A)o(t)17 b(the)h(presen)o(t)f(time,)h(only)g(t)o(w)o(o)f(top)q
(ologies)h(are)f(supp)q(orted)h(for)f(com)o(bines.)28
b(All)19 b(of)e(the)g(notation)150 358 y(used)i(in)g(the)g(discussion)h
(of)e(broadcast)g(top)q(ologies)h(is)g(required)g(in)h(this)f
(discussion.)31 b(In)19 b(addition,)150 415 y(the)d(time)g
Fp(T)360 422 y Fo(o)379 415 y Fs(,)f(de\014ned)i(to)e(b)q(e)h(the)g
(time)g(required)h(to)e(p)q(erform)h(the)f(giv)o(en)i(op)q(eration)f
(\(max,)e(min,)i(or)150 471 y(sum\))f(and)g Fp(T)379
478 y Fo(D)411 471 y Fs(,)g(the)g(time)g(the)h(destination)g(pro)q
(cessor)f(sp)q(ends)h(in)g(the)f(algorithm,)g(are)g(also)g(needed.)150
596 y Fk(F.1)56 b(General)18 b(T)-5 b(ree)18 b(Gather)150
683 y Fs(The)c(\014rst)g(com)o(bine)h(top)q(ology)f(is)h(the)f(general)
h(tree)f(gather,)f(or)h(fan-in,)g(whic)o(h)i(is)e(basically)i(the)e
(same)150 739 y(algorithm)g(as)f(the)h(general)h(tree)f(broadcast)f
(\(or)g(fan-out\))g(describ)q(ed)j(in)f(app)q(endix)g(E.2,)e(except)i
(that)150 795 y(comm)o(unication)g(\015o)o(ws)e(in)i(the)f(opp)q(osite)
h(direction.)21 b(Figures)14 b(11)g(and)g(12)g(sho)o(w)f(the)h(comm)o
(unication)150 852 y(patterns)i(of)g(this)h(algorithm)f(with)h
Fp(N)824 859 y Fo(b)855 852 y Fs(=)e(1)h(and)h Fp(N)1071
859 y Fo(b)1102 852 y Fs(=)e(4)h(\(as)g(b)q(efore,)g
Fp(N)1453 859 y Fo(b)1486 852 y Fs(refers)g(to)g(the)h(n)o(um)o(b)q(er)
f(of)150 908 y(branc)o(hes)g(at)e(eac)o(h)h(no)q(de)h(of)f(the)g
(tree\).)221 965 y(If)k(all)g(pro)q(cessors)g(in)h(the)f(scop)q(e)g(of)
f(the)h(op)q(eration)g(need)h(the)f(information,)g(it)g(is)h
(rebroadcast)150 1022 y(using)g(broadcast's)d(general)j(tree)f
(algorithm.)31 b(This)19 b(top)q(ology)g(can)g(b)q(e)g(called)i(in)f
(the)f(exact)f(same)150 1078 y(w)o(a)o(y)e(as)i(broadcast's)e(general)i
(tree)f(algorithm,)h(i.e.)27 b(through)17 b(the)h(use)g(of)f
Fl(BLACS_SET)f Fs(and)i(setting)150 1135 y Fl(TOP)23
b(=)h('t')p Fs(,)15 b(or)f(b)o(y)h(setting)h Fl(TOP)23
b(='1')15 b Fp(:)8 b(:)g(:)d Fl('9')p Fs(.)221 1192 y(Assuming)12
b(that)f(only)i(one)f(pro)q(cessor)g(needs)h(the)f(answ)o(er)f(\(the)h
(case)g(when)g(all)h(pro)q(cessors)f(require)150 1248
y(the)19 b(answ)o(er)f(will)j(b)q(e)e(dealt)h(with)f(later\))f(this)i
(top)q(ology)e(has)h(man)o(y)f(desirable)j(features.)31
b(First,)19 b(at)150 1305 y(eac)o(h)c(step)f(of)h(the)f(algorithm)h
(only)800 1287 y Ff(1)p 787 1294 44 2 v 787 1320 a Fo(N)815
1326 y Fc(b)850 1305 y Fs(of)g(the)f(pro)q(cessors)h(left)g(in)g(the)g
(op)q(eration)g(go)f(on)g(to)g(the)h(next)150 1361 y(step.)221
1418 y(In)e([16)o(])g(it)g(is)h(sho)o(wn)e(that)h(for)f(the)h(presen)o
(tly)h(supp)q(orted)g(platforms,)e Fp(N)1477 1425 y Fo(b)1506
1418 y Fs(=)h(1)g(will)i(usually)f(b)q(e)g(the)150 1474
y(b)q(est)f(c)o(hoice)h(to)e(minimize)j Fp(T)650 1481
y Fo(D)681 1474 y Fs(.)k(It)13 b(is)g(further)g(demonstrated)f(that)g
Fp(N)1370 1481 y Fo(b)1400 1474 y Fp(>)h Fs(1)f(broadcasts)g(are)h(t)o
(ypically)150 1531 y(comp)q(etitiv)o(e)j(only)g(for)f(small)g(problem)h
(sizes.)221 1588 y(With)22 b(these)f(ca)o(v)o(eats,)h(w)o(e)g(sa)o(y)f
(that)g Fp(N)947 1595 y Fo(b)987 1588 y Fs(=)j(1)d(is)h(the)g(in)o
(teresting)h(c)o(hoice,)g(and)f(then,)i Fp(T)1860 1595
y Fo(D)1915 1588 y Fs(=)150 1644 y Fn(d)p Fp(l)q(og)229
1651 y Ff(2)248 1644 y Fs(\()p Fp(N)303 1651 y Fo(p)322
1644 y Fs(\))p Fn(e)p Fs(\()p Fp(T)405 1651 y Fo(c)434
1644 y Fs(+)13 b Fp(T)509 1651 y Fo(o)528 1644 y Fs(\).)31
b(If)19 b(all)h(pro)q(cessors)f(require)h(the)f(answ)o(er,)g(it)g(is)g
(found)h(as)e(ab)q(o)o(v)o(e,)h(and)h(then)150 1701 y(broadcast)e(to)h
(all)h(pro)q(cessors)f(via)g(the)g(general)h(tree)f(algorithm)g
(describ)q(ed)i(in)f(Section)g(E.2.)31 b(The)150 1757
y(longest)10 b(time)h(an)o(y)f(pro)q(cessor)g(w)o(ould)h(then)f(sp)q
(end)i(in)f(the)f(algorithm)g(w)o(ould)h(b)q(e)g Fn(d)p
Fp(l)q(og)1612 1764 y Ff(2)1631 1757 y Fs(\()p Fp(N)1686
1764 y Fo(p)1705 1757 y Fs(\))p Fn(e)p Fs(\(2)p Fn(\003)p
Fp(T)1834 1764 y Fo(c)1851 1757 y Fs(+)p Fp(T)1913 1764
y Fo(o)1932 1757 y Fs(\))221 1814 y(This)20 b(top)q(ology)g(is)h(alw)o
(a)o(ys)e(coheren)o(t,)j(and)e(b)o(y)g(default)h(is)g(not)f(rep)q
(eatable.)35 b(It)21 b(can)f(b)q(e)h(made)150 1871 y(rep)q(eatable)16
b(b)o(y)f(forcing)h(an)f(ordering)g(on)g(the)h(reciev)o(es.)150
1995 y Fk(F.2)56 b(Bidirectional)16 b(Exc)n(hange)150
2082 y Fs(This)f(top)q(ology)g(is)g(sp)q(ecialized)j(for)c(lea)o(v)o
(e-on-all)i(com)o(bines,)f(and)g(therefore,)f(if)h(a)g(lea)o(v)o
(e-on-one)g(com-)150 2139 y(bine)21 b(has)e(b)q(een)h(requested,)g(the)
g(general)f(tree)g(algorithm)h(with)f Fp(N)1364 2146
y Fo(b)1400 2139 y Fs(=)h(1)f(is)h(called)h(instead.)32
b(It)19 b(is)150 2195 y(based)e(on)h(an)f(algorithm)g(presen)o(ted)h
(in)g([3)o(].)25 b(This)18 b(top)q(ology)f(in)o(v)o(olv)o(es)h(ha)o
(ving)f(pairs)g(of)g(pro)q(cessors)150 2251 y(exc)o(hange)i
(information,)g(and)g(th)o(us)f(it)h(p)q(erforms)f(b)q(est)h(when)g
Fp(N)1308 2258 y Fo(p)1346 2251 y Fs(is)g(an)f(in)o(teger)h(p)q(o)o(w)o
(er)f(of)g(2.)30 b(The)150 2308 y(comm)o(unication)18
b(pattern)e(inheren)o(t)i(in)f(this)h(algorithm)e(is)i(sho)o(wn)e(in)i
(Figure)f(13.)24 b(As)17 b(the)g(user)g(can)150 2364
y(see,)e(this)g(an)g(extremely)g(\\noisy")g(algorithm:)20
b(ev)o(ery)14 b(pro)q(cessor)h(is)g(sending)h(and)f(receiving)i(at)d
(ev)o(ery)150 2421 y(step)h(in)h(the)g(algorithm.)j(It)d(is)f(called)i
(b)o(y)e(setting)h Fl(TOP)23 b(=)h('h')p Fs(.)221 2478
y(Unless)15 b(the)g(platform)f(supp)q(orts)h(the)g(o)o(v)o(erlap)f(of)h
(sends)g(and)g(receiv)o(es,)g(this)g(top)q(ology)g(is)g(inferior)150
2534 y(to)e(fan-in/fan-out.)19 b(If)14 b(sends)g(and)g(receiv)o(es)g
(cannot)g(o)q(ccur)g(sim)o(ultaneously)l(,)h(the)e(b)q(est)h(sp)q(eed)h
(this)e(al-)150 2591 y(gorithm)e(can)h(ac)o(hiev)o(e)g(is)g
Fp(T)621 2598 y Fo(D)666 2591 y Fs(=)h Fp(l)q(og)773
2598 y Ff(2)791 2591 y Fs(\()p Fp(N)846 2598 y Fo(p)865
2591 y Fs(\))s Fn(\003)s Fs(\(2)s Fn(\003)s Fp(T)1009
2598 y Fo(c)1028 2591 y Fs(+)s Fp(T)1093 2598 y Fo(o)1112
2591 y Fs(\),)e(the)h(same)f(as)g(for)g(fan-in/fan-out.)19
b(Ho)o(w)o(ev)o(er,)150 2647 y(fan-in/fan-out)c(in)g(general)g(has)g
(less)g(link)h(con)o(ten)o(tion,)f(so)f(this)h(top)q(ology)f(is)h(only)
h(recommended)f(on)150 2704 y(platforms)g(where)g(sends)h(and)f(receiv)
o(es)h(can)g(b)q(e)f(o)o(v)o(erlapp)q(ed.)1027 2828 y(56)p
eop
%%Page: 57 61
57 60 bop 414 409 a Fb(\032\031)414 293 y(\033\030)584
409 y(\032\031)584 293 y(\033\030)754 409 y(\032\031)754
293 y(\033)q(\030)925 409 y(\032\031)925 293 y(\033\030)1095
409 y(\032\031)1095 293 y(\033\030)1265 409 y(\032\031)1265
293 y(\033\030)1435 409 y(\032\031)1435 293 y(\033\030)1605
409 y(\032\031)1605 293 y(\033)q(\030)414 708 y(\032\031)414
592 y(\033\030)754 708 y(\032\031)754 592 y(\033)q(\030)1095
708 y(\032\031)1095 592 y(\033\030)1435 708 y(\032\031)1435
592 y(\033\030)414 1007 y(\032\031)414 891 y(\033\030)1095
1007 y(\032\031)1095 891 y(\033\030)414 1306 y(\032\031)414
1190 y(\033\030)192 362 y Fp(S)15 b Fs(=)e(0)192 661
y Fp(S)i Fs(=)e(1)192 960 y Fp(S)i Fs(=)e(2)192 1259
y Fp(S)i Fs(=)e(3)461 362 y(0)461 661 y(0)461 960 y(0)461
1259 y(0)632 362 y(1)147 b(2)802 661 y(2)972 362 y(3)g(4)1142
661 y(4)1142 960 y(4)1312 362 y(5)h(6)1483 661 y(6)1653
362 y(7)613 449 y Fa(\023)582 491 y(\023)551 532 y(\023)519
574 y(\023)503 596 y(\023)-31 b(/)953 449 y(\023)922
491 y(\023)891 532 y(\023)860 574 y(\023)843 596 y(\023)g(/)1294
449 y(\023)1262 491 y(\023)1231 532 y(\023)1200 574 y(\023)1184
596 y(\023)g(/)1634 449 y(\023)1603 491 y(\023)1572 532
y(\023)1541 574 y(\023)1524 596 y(\023)g(/)773 734 y(\021)731
762 y(\021)690 790 y(\021)648 817 y(\021)607 845 y(\021)565
873 y(\021)524 900 y(\021)515 906 y(\021)-42 b(+)1453
734 y(\021)1412 762 y(\021)1370 790 y(\021)1329 817 y(\021)1287
845 y(\021)1246 873 y(\021)1204 900 y(\021)1196 906 y(\021)g(+)1113
1019 y(\020)1072 1033 y(\020)1030 1047 y(\020)989 1061
y(\020)947 1075 y(\020)906 1088 y(\020)864 1102 y(\020)822
1116 y(\020)781 1130 y(\020)739 1144 y(\020)698 1158
y(\020)656 1171 y(\020)615 1185 y(\020)573 1199 y(\020)532
1213 y(\020)524 1216 y(\020)g(\))618 1419 y Fs(Figure)15
b(11:)k(General)d(tree)f(gather)g(with)g Fp(N)1382 1426
y Fo(b)1412 1419 y Fs(=)e(1)414 1855 y Fb(\032\031)414
1738 y(\033\030)584 1855 y(\032\031)584 1738 y(\033\030)754
1855 y(\032\031)754 1738 y(\033)q(\030)925 1855 y(\032\031)925
1738 y(\033\030)1095 1855 y(\032\031)1095 1738 y(\033\030)1265
1855 y(\032\031)1265 1738 y(\033\030)1435 1855 y(\032\031)1435
1738 y(\033\030)1605 1855 y(\032\031)1605 1738 y(\033)q(\030)414
2153 y(\032\031)414 2037 y(\033\030)1265 2153 y(\032\031)1265
2037 y(\033\030)414 2452 y(\032\031)414 2336 y(\033\030)192
1807 y Fp(S)i Fs(=)e(0)192 2106 y Fp(S)i Fs(=)e(1)192
2405 y Fp(S)i Fs(=)e(2)461 1807 y(0)461 2106 y(0)461
2405 y(0)632 1807 y(1)147 b(2)g(3)g(4)g(5)1312 2106 y(5)1483
1807 y(6)g(7)613 1894 y Fa(\023)582 1936 y(\023)551 1977
y(\023)519 2019 y(\023)503 2041 y(\023)-31 b(/)1464 1894
y(\023)1433 1936 y(\023)1402 1977 y(\023)1370 2019 y(\023)1354
2041 y(\023)g(/)773 1881 y(\021)731 1908 y(\021)690 1936
y(\021)648 1964 y(\021)607 1991 y(\021)565 2019 y(\021)524
2047 y(\021)515 2052 y(\021)-42 b(+)1624 1881 y(\021)1582
1908 y(\021)1541 1936 y(\021)1499 1964 y(\021)1458 1991
y(\021)1416 2019 y(\021)1375 2047 y(\021)1366 2052 y(\021)g(+)943
1874 y(\010)901 1894 y(\010)860 1915 y(\010)818 1936
y(\010)777 1957 y(\010)735 1977 y(\010)694 1998 y(\010)652
2019 y(\010)611 2040 y(\010)569 2060 y(\010)536 2077
y(\010)g(\031)1113 1867 y(\020)1072 1881 y(\020)1030
1894 y(\020)989 1908 y(\020)947 1922 y(\020)906 1936
y(\020)864 1950 y(\020)822 1964 y(\020)781 1977 y(\020)739
1991 y(\020)698 2005 y(\020)656 2019 y(\020)615 2033
y(\020)573 2047 y(\020)532 2060 y(\020)528 2062 y(\020)g(\))1283
2162 y(\030)1242 2173 y(\030)1200 2183 y(\030)1159 2193
y(\030)1117 2204 y(\030)1076 2214 y(\030)1034 2224 y(\030)993
2235 y(\030)951 2245 y(\030)910 2256 y(\030)868 2266
y(\030)827 2276 y(\030)785 2287 y(\030)744 2297 y(\030)702
2307 y(\030)661 2318 y(\030)619 2328 y(\030)578 2339
y(\030)536 2349 y(\030)515 2354 y(\030)g(9)618 2565 y
Fs(Figure)15 b(12:)k(General)d(tree)f(gather)g(with)g
Fp(N)1382 2572 y Fo(b)1412 2565 y Fs(=)e(4)1027 2828
y(57)p eop
%%Page: 58 62
58 61 bop 414 1135 a Fb(\032\031)414 1019 y(\033\030)584
1135 y(\032\031)584 1019 y(\033\030)754 1135 y(\032\031)754
1019 y(\033)q(\030)925 1135 y(\032\031)925 1019 y(\033\030)1095
1135 y(\032\031)1095 1019 y(\033\030)1265 1135 y(\032\031)1265
1019 y(\033\030)1435 1135 y(\032\031)1435 1019 y(\033\030)1605
1135 y(\032\031)1605 1019 y(\033)q(\030)414 720 y(\032\031)414
604 y(\033\030)584 720 y(\032\031)584 604 y(\033\030)754
720 y(\032\031)754 604 y(\033)q(\030)925 720 y(\032\031)925
604 y(\033\030)1095 720 y(\032\031)1095 604 y(\033\030)1265
720 y(\032\031)1265 604 y(\033\030)1435 720 y(\032\031)1435
604 y(\033\030)1605 720 y(\032\031)1605 604 y(\033)q(\030)414
305 y(\032\031)414 188 y(\033\030)584 305 y(\032\031)584
188 y(\033\030)754 305 y(\032\031)754 188 y(\033)q(\030)925
305 y(\032\031)925 188 y(\033\030)1095 305 y(\032\031)1095
188 y(\033\030)1265 305 y(\032\031)1265 188 y(\033\030)1435
305 y(\032\031)1435 188 y(\033\030)1605 305 y(\032\031)1605
188 y(\033)q(\030)192 257 y Fp(S)15 b Fs(=)e(0)192 672
y Fp(S)i Fs(=)e(1)192 1088 y Fp(S)i Fs(=)e(2)155 b(0)461
672 y(0)461 257 y(0)632 1088 y(1)632 672 y(1)632 257
y(1)802 1088 y(2)802 672 y(2)802 257 y(2)972 1088 y(3)972
672 y(3)972 257 y(3)1142 1088 y(4)1142 672 y(4)1142 257
y(4)1312 1088 y(5)1312 672 y(5)1312 257 y(5)1483 1088
y(6)1483 672 y(6)1483 257 y(6)1653 1088 y(7)1653 672
y(7)1653 257 y(7)517 198 y Fa(\021Q)-42 b(s)257 b(\021)o(Q)-42
b(s)257 b(\021Q)-42 b(s)257 b(\021)o(Q)-42 b(s)517 320
y(Q)g(k\021)257 b(Q)-42 b(k)o(\021)257 b(Q)-42 b(k\021)257
b(Q)-42 b(k)o(\021)602 576 y(\021)561 603 y(\021)524
628 y(\021)773 576 y(\021)731 603 y(\021)694 628 y(\021)644
576 y(Q)685 603 y(Q)723 628 y(Q)g(s)814 576 y(Q)856 603
y(Q)893 628 y(Q)g(s)602 772 y(Q)561 744 y(Q)524 720 y(Q)g(k)773
772 y(Q)731 744 y(Q)694 720 y(Q)g(k)644 772 y(\021)685
744 y(\021)723 720 y(\021)814 772 y(\021)856 744 y(\021)893
720 y(\021)1283 576 y(\021)1242 603 y(\021)1204 628 y(\021)1453
576 y(\021)1412 603 y(\021)1375 628 y(\021)1325 576 y(Q)1366
603 y(Q)1404 628 y(Q)g(s)1495 576 y(Q)1536 603 y(Q)1574
628 y(Q)g(s)1283 772 y(Q)1242 744 y(Q)1204 720 y(Q)g(k)1453
772 y(Q)1412 744 y(Q)1375 720 y(Q)g(k)1325 772 y(\021)1366
744 y(\021)1404 720 y(\021)1495 772 y(\021)1536 744 y(\021)1574
720 y(\021)773 901 y(\010)731 922 y(\010)690 942 y(\010)648
963 y(\010)607 984 y(\010)565 1005 y(\010)524 1025 y(\010)515
1029 y(\010)943 901 y(\010)901 922 y(\010)860 942 y(\010)818
963 y(\010)777 984 y(\010)735 1005 y(\010)694 1025 y(\010)685
1029 y(\010)1113 901 y(\010)1072 922 y(\010)1030 942
y(\010)989 963 y(\010)947 984 y(\010)906 1005 y(\010)864
1025 y(\010)856 1029 y(\010)1283 901 y(\010)1242 922
y(\010)1200 942 y(\010)1159 963 y(\010)1117 984 y(\010)1076
1005 y(\010)1034 1025 y(\010)1026 1029 y(\010)814 901
y(H)856 922 y(H)897 942 y(H)939 963 y(H)980 984 y(H)1022
1005 y(H)1063 1025 y(H)1072 1029 y(H)g(j)984 901 y(H)1026
922 y(H)1067 942 y(H)1109 963 y(H)1150 984 y(H)1192 1005
y(H)1233 1025 y(H)1242 1029 y(H)g(j)1155 901 y(H)1196
922 y(H)1238 942 y(H)1279 963 y(H)1321 984 y(H)1362 1005
y(H)1404 1025 y(H)1412 1029 y(H)g(j)1325 901 y(H)1366
922 y(H)1408 942 y(H)1449 963 y(H)1491 984 y(H)1532 1005
y(H)1574 1025 y(H)1582 1029 y(H)g(j)773 1270 y(H)731
1249 y(H)690 1229 y(H)648 1208 y(H)607 1187 y(H)565 1166
y(H)524 1146 y(H)515 1142 y(H)g(Y)943 1270 y(H)901 1249
y(H)860 1229 y(H)818 1208 y(H)777 1187 y(H)735 1166 y(H)694
1146 y(H)685 1142 y(H)g(Y)1113 1270 y(H)1072 1249 y(H)1030
1229 y(H)989 1208 y(H)947 1187 y(H)906 1166 y(H)864 1146
y(H)856 1142 y(H)g(Y)1283 1270 y(H)1242 1249 y(H)1200
1229 y(H)1159 1208 y(H)1117 1187 y(H)1076 1166 y(H)1034
1146 y(H)1026 1142 y(H)g(Y)814 1270 y(\010)856 1249 y(\010)897
1229 y(\010)939 1208 y(\010)980 1187 y(\010)1022 1166
y(\010)1063 1146 y(\010)1072 1142 y(\010)984 1270 y(\010)1026
1249 y(\010)1067 1229 y(\010)1109 1208 y(\010)1150 1187
y(\010)1192 1166 y(\010)1233 1146 y(\010)1242 1142 y(\010)1155
1270 y(\010)1196 1249 y(\010)1238 1229 y(\010)1279 1208
y(\010)1321 1187 y(\010)1362 1166 y(\010)1404 1146 y(\010)1412
1142 y(\010)1325 1270 y(\010)1366 1249 y(\010)1408 1229
y(\010)1449 1208 y(\010)1491 1187 y(\010)1532 1166 y(\010)1574
1146 y(\010)1582 1142 y(\010)715 1372 y Fs(Figure)15
b(13:)k(Bidirectional)f(exc)o(hange)221 1518 y(Assuming)i(sim)o
(ultaneous)h(send)f(and)g(receiv)o(e,)i(w)o(e)d(ha)o(v)o(e)h(t)o(w)o(o)
e(in)o(teresting)j(cases.)33 b(If)21 b Fp(N)1813 1525
y Fo(p)1852 1518 y Fs(is)f(an)150 1574 y(in)o(teger)f(p)q(o)o(w)o(er)f
(of)h(t)o(w)o(o,)e(all)j(pro)q(cessors)f(will)h(sp)q(end)g(roughly)f
Fp(T)1305 1581 y Fo(D)1355 1574 y Fs(=)g Fp(l)q(og)1468
1581 y Ff(2)1487 1574 y Fs(\()p Fp(N)1542 1581 y Fo(p)1561
1574 y Fs(\))12 b Fn(\003)g Fs(\()p Fp(T)1671 1581 y
Fo(c)1700 1574 y Fs(+)h Fp(T)1775 1581 y Fo(o)1794 1574
y Fs(\))18 b(in)i(the)150 1631 y(algorithm.)32 b(If)20
b Fp(N)474 1638 y Fo(p)512 1631 y Fs(is)g(not)f(an)g(in)o(teger)g(p)q
(o)o(w)o(er)g(of)g(t)o(w)o(o,)f(the)i(\014rst)f(step)g(of)g(the)g
(algorithm)g(requires)150 1687 y(pro)q(cessors)g(b)q(ey)o(ond)h(the)g
(p)q(o)o(w)o(er)f(of)g(t)o(w)o(o)f(to)h(send)i(their)f(v)m(alues)g(to)f
(pro)q(cessors)h(within)g(an)g(in)o(teger)150 1744 y(p)q(o)o(w)o(er)h
(of)g(t)o(w)o(o,)g(the)g(normal)h(bidirectional)h(exc)o(hange)f(tak)o
(es)f(place,)i(and)f(then)f(the)h(answ)o(ers)e(are)150
1800 y(sen)o(t)g(bac)o(k)g(out)g(to)f(the)i(non-p)q(o)o(w)o(er)f(of)g
(t)o(w)o(o)e(pro)q(cessors.)35 b(Then,)22 b(pro)q(cessors)d(will)j(sp)q
(end)g(roughly)150 1856 y Fp(T)177 1863 y Fo(D)221 1856
y Fs(=)13 b(2)d Fn(\003)g Fp(T)362 1863 y Fo(c)389 1856
y Fs(+)g Fp(T)461 1863 y Fo(o)490 1856 y Fs(+)h Fn(b)p
Fp(l)q(og)615 1863 y Ff(2)634 1856 y Fs(\()p Fp(N)689
1863 y Fo(p)708 1856 y Fs(\))p Fn(c)f(\003)f Fs(\()p
Fp(T)833 1863 y Fo(c)860 1856 y Fs(+)i Fp(T)933 1863
y Fo(o)951 1856 y Fs(\))k(in)h(the)f(algorithm.)221 1917
y(In)e(the)f(b)q(est)h(case,)g(this)f(algorithm)h(will)h(giv)o(e)f(all)
g(pro)q(cessors)f(the)h(answ)o(er)f(in)h(the)g(same)f(amoun)o(t)f(of)
150 1973 y(time)i(that)f(it)g(tak)o(es)g(to)g(get)g(the)h(answ)o(er)f
(to)g(one)g(pro)q(cessor)h(using)g(the)g(fan-in)g(algorithm.)19
b(Ho)o(w)o(ev)o(er,)12 b(it)150 2030 y(will)k(rarely)e(b)q(e)h(the)f
(case)g(that)g(this)g(sp)q(eed)h(is)g(realized.)21 b(Not)14
b(only)g(m)o(ust)g(sim)o(ultaneous)h(send/receiv)o(e)150
2086 y(b)q(e)j(allo)o(w)o(ed,)g(but)g(a)f(t)o(wice)g(the)h(bandwidth)g
(is)g(required,)h(and)e(a)g(net)o(w)o(ork)f(of)h(at)g(least)h(the)f
(ric)o(hness)150 2143 y(of)e(a)f(h)o(yp)q(ercub)q(e)j(is)e(required)i
(to)d(a)o(v)o(oid)h(link)h(con\015icts.)21 b(Therefore,)14
b(fan-in/fan-out)i(should)g(b)q(e)f(used)150 2199 y(in)h(the)g(general)
g(case,)f(and)h(this)f(top)q(ology)h(should)g(b)q(e)g(utilized)i(only)e
(when)g(timings)g(sho)o(w)f(that)g(it)g(is)150 2256 y(sup)q(erior.)221
2316 y(This)f(top)q(ology)g(is)h(homogeneous)f(coheren)o(t,)g(and)h
(rep)q(eatable.)20 b(If)15 b(the)f(user)h(asks)e(the)i(BLA)o(CS)g(to)
150 2372 y(enforce)g(heterogeneous)h(coherence,)g(this)f(top)q(ology)g
(will)i(not)e(b)q(e)h(used.)150 2538 y Fr(G)69 b(Multiring)21
b(Com)n(bine)150 2647 y Fs(In)c(this)g(release,)g(only)g(the)f(MPI)g(v)
o(ersion)h(has)f(a)g(m)o(ultiring)i(com)o(bine.)24 b(This)17
b(top)q(ology)f(is)g(m)o(uc)o(h)h(lik)o(e)150 2704 y(m)o(ultiring)12
b(broadcast:)17 b(the)11 b(pro)q(cesses)h(participating)g(in)g(the)f
(com)o(bine)h(are)f(split)h(up)f(in)o(to)g Fp(N)1752
2711 y Fo(r)1782 2704 y Fs(\(n)o(um)o(b)q(er)1027 2828
y(58)p eop
%%Page: 59 63
59 62 bop 150 199 a Fs(of)18 b(rings)h(set)f(b)o(y)g(the)h(user\))f
(sep)q(erate)g(increasing)i(or)e(decreasing)h(rings,)g(whic)o(h)h(send)
f(their)g(partial)150 256 y(result)c(to)g(the)g(destination)g(pro)q
(cess,)g(where)g(the)g(\014nal)h(result)f(is)h(calculated.)21
b(If)15 b(the)g(answ)o(er)g(is)g(to)f(b)q(e)150 312 y(left)i(on)f(all)h
(pro)q(cesses,)f(it)g(will)i(b)q(e)f(broadcast)f(using)h(the)f(m)o
(ultiring)h(broadcast.)221 369 y(The)e(time)h(to)f(get)g(the)g(answ)o
(er)g(to)g(one)g(pro)q(cess)h(is)g Fp(T)1140 376 y Fo(D)1184
369 y Fs(=)e(\()p Fn(b)p Fs(\()p Fp(N)1325 376 y Fo(p)1352
369 y Fn(\000)c Fs(1\))p Fp(=)m(N)1494 376 y Fo(r)1512
369 y Fn(c)g Fs(+)g Fp(N)1622 376 y Fo(r)1649 369 y Fn(\000)g
Fs(1\))e Fn(\003)i Fs(\()p Fp(T)1818 376 y Fo(c)1843
369 y Fs(+)g Fp(T)1914 376 y Fo(o)1932 369 y Fs(\))150
425 y(If)k(the)h(answ)o(er)e(is)i(left)g(on)f(all)h(no)q(des,)g(the)f
(longest)h(time)f(an)o(y)g(pro)q(cess)h(sp)q(end)g(in)g(the)f
(algorithm)h(w)o(ould)150 482 y(b)q(e)i(\()p Fn(b)p Fs(\()p
Fp(N)305 489 y Fo(p)334 482 y Fn(\000)10 b Fs(1\))p Fp(=)m(N)477
489 y Fo(r)495 482 y Fn(c)h Fs(+)f Fp(N)608 489 y Fo(r)637
482 y Fn(\000)g Fs(1\))g Fn(\003)g Fs(\(2)f Fn(\003)h
Fp(T)876 489 y Fo(c)903 482 y Fs(+)g Fp(T)975 489 y Fo(o)994
482 y Fs(\).)221 538 y(This)i(algorithm)f(can)h(b)q(e)g(used)g(to)f
(minimize)j(link)f(con)o(ten)o(tion)e(on)h(systems)f(where)g(that)g(is)
h(a)f(ma)s(jor)150 594 y(concern,)k(and)h(it)f(can)h(displa)o(y)g(pip)q
(elini)q(ng)i(as)d(w)o(ell.)221 651 y(Com)o(bine)e(pip)q(elin)q(ing)k
(is)d(not)f(as)g(straigh)o(tforw)o(ard)e(as)i(broadcast.)19
b(First,)13 b(if)h(the)f(answ)o(er)g(is)h(left)g(on)150
707 y(all)k(pro)q(cesses,)g(the)g(maximal)f(pip)q(e)i(length)f(will)h
(b)q(e)g Fn(b)p Fs(\()p Fp(N)1166 714 y Fo(p)1196 707
y Fn(\000)12 b Fs(1\))p Fp(=)m(N)1341 714 y Fo(r)1359
707 y Fn(c)p Fs(.)27 b(After)17 b(this)h(man)o(y)f(com)o(bines,)150
764 y(the)e(broadcast)g(message)f(from)h(previous)h(com)o(bines)g(will)
h(b)q(egin)f(to)f(in)o(terfere)g(with)h(new)f(com)o(bines.)221
820 y(Mixing)e(pip)q(es)h(formed)f(b)o(y)f(broadcasts)g(with)h(those)f
(made)h(b)o(y)g(com)o(bines)g(is)g(not)f(straigh)o(tforw)o(ard.)150
877 y(In)19 b(order)f(to)g(use)g(a)g(pip)q(e)i(made)e(b)o(y)h(an)f
(increasing)h(ring)g(broadcast,)f(for)g(instance,)h(the)g(com)o(bine's)
150 933 y(destination)k(pro)q(cess)f(m)o(ust)g(b)q(e)g(the)g(left)h
(neigh)o(b)q(or)f(of)g(the)g(broadcast's)f(source.)40
b(Similarly)l(,)26 b(the)150 990 y(com)o(bine)16 b(destination)g(w)o
(ould)g(need)g(to)e(b)q(e)i(the)f(righ)o(t)g(neigh)o(b)q(er)i(of)e(a)g
(decreasing)h(broadcast)e(source.)1027 2828 y(59)p eop
%%Page: 60 64
60 63 bop 150 199 a Fr(H)69 b(Example)21 b(Program)150
301 y Fs(The)12 b(follo)o(wing)h(routine)f(tak)o(es)f(the)h(a)o(v)m
(ailable)h(pro)q(cesses,)g(forms)e(them)h(in)o(to)g(a)f(pro)q(cess)h
(grid,)h(and)f(then)150 357 y(has)k(eac)o(h)g(pro)q(cess)g(c)o(hec)o(k)
h(in)g(with)f(the)g(pro)q(cess)g(at)g Fn(f)p Fs(0,0)p
Fn(g)e Fs(in)j(the)f(pro)q(cess)h(grid.)22 b(F)l(or)16
b(more)f(detailed)150 414 y(examples,)h(see)f(the)g(BLA)o(CS)h
(homepage.)293 502 y Fl(PROGRAM)23 b(HELLO)150 558 y(*)119
b(--)24 b(BLACS)f(example)g(code)g(--)150 615 y(*)119
b(Written)23 b(by)h(Clint)f(Whaley)g(7/26/94)150 671
y(*)119 b(Performs)23 b(a)h(simple)f(check-in)f(type)i(hello)f(world)
150 728 y(*)119 b(..)150 784 y(*)g(..)24 b(External)f(Functions)f(..)
293 841 y(INTEGER)h(BLACS_PNUM)293 897 y(EXTERNAL)g(BLACS_PNUM)150
953 y(*)119 b(..)150 1010 y(*)g(..)24 b(Variable)f(Declaration)f(..)293
1066 y(INTEGER)h(CONTXT,)g(IAM,)g(NPROCS,)g(NPROW,)g(NPCOL,)g(MYPROW,)g
(MYPCOL)293 1123 y(INTEGER)g(ICALLER,)g(I,)g(J,)h(HISROW,)f(HISCOL)150
1179 y(*)150 1236 y(*)119 b(Determine)23 b(my)g(process)g(number)g(and)
h(the)f(number)g(of)h(processes)f(in)150 1292 y(*)119
b(machine)150 1349 y(*)293 1405 y(CALL)23 b(BLACS_PINFO\(IAM,)f
(NPROCS\))150 1462 y(*)150 1518 y(*)119 b(If)24 b(in)f(PVM,)h(create)f
(virtual)g(machine)g(if)g(it)h(doesn't)f(exist)150 1574
y(*)293 1631 y(IF)h(\(NPROCS)f(.LT.)g(1\))h(THEN)365
1687 y(IF)f(\(IAM)h(.EQ.)f(0\))h(THEN)436 1744 y(WRITE\(*,)f(1000\))436
1800 y(READ\(*,)g(2000\))g(NPROCS)365 1857 y(END)g(IF)365
1913 y(CALL)g(BLACS_SETUP\(IAM,)f(NPROCS\))293 1970 y(END)i(IF)150
2026 y(*)150 2083 y(*)119 b(Set)24 b(up)f(process)g(grid)g(that)h(is)f
(as)h(close)f(to)h(square)f(as)g(possible)150 2139 y(*)293
2195 y(NPROW)g(=)h(INT\()f(SQRT\()h(REAL\(NPROCS\))e(\))i(\))293
2252 y(NPCOL)f(=)h(NPROCS)f(/)h(NPROW)150 2308 y(*)150
2365 y(*)119 b(Get)24 b(default)f(system)g(context,)f(and)i(define)f
(grid)150 2421 y(*)293 2478 y(CALL)g(BLACS_GET\(0,)g(0,)g(CONTXT\))293
2534 y(CALL)g(BLACS_GRIDINIT\(CONTXT,)e('ROW',)i(NPROW,)g(NPCOL\))293
2591 y(CALL)g(BLACS_GRIDINFO\(CONTXT,)e(NPROW,)i(NPCOL,)g(MYPROW,)g
(MYPCOL\))150 2647 y(*)150 2704 y(*)119 b(If)24 b(I'm)f(not)h(in)f
(grid,)g(go)h(to)g(end)f(of)h(program)1027 2828 y Fs(60)p
eop
%%Page: 61 65
61 64 bop 150 199 a Fl(*)293 256 y(IF)24 b(\()f(\(MYPROW.GE.NPROW\))f
(.OR.)h(\(MYPCOL.GE.NPCOL\))f(\))i(GOTO)f(30)150 312
y(*)150 369 y(*)119 b(Get)24 b(my)f(process)g(ID)h(from)f(my)h(grid)f
(coordinates)150 425 y(*)293 482 y(ICALLER)g(=)h(BLACS_PNUM\(CONTXT,)d
(MYPROW,)i(MYPCOL\))150 538 y(*)150 594 y(*)119 b(If)24
b(I)f(am)h(process)f({0,0},)g(receive)g(check-in)g(messages)f(from)150
651 y(*)119 b(all)24 b(nodes)150 707 y(*)293 764 y(IF)g(\()f
(\(MYPROW.EQ.0\))g(.AND.)g(\(MYPCOL.EQ.0\))f(\))i(THEN)365
877 y(WRITE\(*,*\))e(')i(')365 933 y(DO)f(20)h(I)g(=)f(0,)h(NPROW-1)436
990 y(DO)g(10)g(J)f(=)h(0,)g(NPCOL-1)508 1103 y(IF)g(\()f(\(I.NE.0\))g
(.OR.)g(\(J.NE.0\))g(\))h(THEN)580 1159 y(CALL)f(IGERV2D\(CONTXT,)f(1,)
h(1,)h(ICALLER,)f(1,)g(I,)h(J\))508 1215 y(ENDIF)150
1272 y(*)150 1328 y(*)334 b(Make)23 b(sure)h(ICALLER)e(is)i(where)f(we)
h(think)f(in)h(process)f(grid)150 1385 y(*)508 1441 y(CALL)g
(BLACS_PCOORD\(CONTXT,)e(ICALLER,)i(HISROW,)g(HISCOL\))508
1498 y(IF)h(\()f(\(HISROW.NE.I\))f(.OR.)i(\(HISCOL.NE.J\))e(\))h(THEN)
580 1554 y(WRITE\(*,*\))f('Grid)h(error!)47 b(Halting)23
b(.)h(.)f(.')580 1611 y(STOP)508 1667 y(END)g(IF)508
1724 y(WRITE\(*,)g(3000\))g(I,)g(J,)h(ICALLER)150 1836
y(10)238 b(CONTINUE)150 1893 y(20)167 b(CONTINUE)365
1949 y(WRITE\(*,*\))22 b(')i(')365 2006 y(WRITE\(*,*\))e('All)i
(processes)e(checked)h(in.)47 b(Run)24 b(finished.')150
2062 y(*)150 2119 y(*)119 b(All)24 b(processes)e(but)i({0,0})f(send)g
(process)g(ID)h(as)f(a)h(check-in)150 2175 y(*)293 2232
y(ELSE)365 2288 y(CALL)f(IGESD2D\(CONTXT,)f(1,)h(1,)h(ICALLER,)f(1,)g
(0,)h(0\))293 2345 y(END)g(IF)150 2457 y(30)95 b(CONTINUE)293
2570 y(CALL)23 b(BLACS_EXIT\(0\))150 2683 y(1000)47 b(FORMAT\('How)23
b(many)g(processes)f(in)i(machine?'\))1027 2828 y Fs(61)p
eop
%%Page: 62 66
62 65 bop 150 199 a Fl(2000)47 b(FORMAT\(I\))150 256
y(3000)g(FORMAT\('Process)22 b({',i2,',',i2,'})g(\(node)h(number)g
(=',I,)269 312 y($)167 b('\))24 b(has)f(checked)g(in.'\))293
425 y(STOP)293 482 y(END)1027 2828 y Fs(62)p eop
%%Trailer
end
userdict /end-hook known{end-hook}if
%%EOF