X Tutup
Skip to content

Commit ebc871a

Browse files
nhanfordnhanford
authored
[WIP] relocate.py: parallelize test replacement logic (spack#19690)
* sbang pushed back to callers; star moved to util.lang * updated unit test * sbang test moved; local tests pass Co-authored-by: Nathan Hanford <hanford1@llnl.gov>
1 parent c63f680 commit ebc871a

File tree

10 files changed

+316
-210
lines changed

10 files changed

+316
-210
lines changed

‎lib/spack/llnl/util/lang.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,13 @@ def uniq(sequence):
673673
return uniq_list
674674

675675

676+
def star(func):
677+
"""Unpacks arguments for use with Multiprocessing mapping functions"""
678+
def _wrapper(args):
679+
return func(*args)
680+
return _wrapper
681+
682+
676683
class Devnull(object):
677684
"""Null stream with less overhead than ``os.devnull``.
678685

‎lib/spack/spack/binary_distribution.py‎

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import tempfile
1313
import hashlib
1414
import glob
15+
from ordereddict_backport import OrderedDict
1516

1617
from contextlib import closing
1718
import ruamel.yaml as yaml
@@ -1105,11 +1106,26 @@ def relocate_package(spec, allow_root):
11051106
new_deps = spack.build_environment.get_rpath_deps(spec.package)
11061107
for d in new_deps:
11071108
hash_to_prefix[d.format('{hash}')] = str(d.prefix)
1108-
prefix_to_prefix = dict()
1109+
# Spurious replacements (e.g. sbang) will cause issues with binaries
1110+
# For example, the new sbang can be longer than the old one.
1111+
# Hence 2 dictionaries are maintained here.
1112+
prefix_to_prefix_text = OrderedDict({})
1113+
prefix_to_prefix_bin = OrderedDict({})
1114+
prefix_to_prefix_text[old_prefix] = new_prefix
1115+
prefix_to_prefix_bin[old_prefix] = new_prefix
1116+
prefix_to_prefix_text[old_layout_root] = new_layout_root
1117+
prefix_to_prefix_bin[old_layout_root] = new_layout_root
11091118
for orig_prefix, hash in prefix_to_hash.items():
1110-
prefix_to_prefix[orig_prefix] = hash_to_prefix.get(hash, None)
1111-
prefix_to_prefix[old_prefix] = new_prefix
1112-
prefix_to_prefix[old_layout_root] = new_layout_root
1119+
prefix_to_prefix_text[orig_prefix] = hash_to_prefix.get(hash, None)
1120+
prefix_to_prefix_bin[orig_prefix] = hash_to_prefix.get(hash, None)
1121+
# This is vestigial code for the *old* location of sbang. Previously,
1122+
# sbang was a bash script, and it lived in the spack prefix. It is
1123+
# now a POSIX script that lives in the install prefix. Old packages
1124+
# will have the old sbang location in their shebangs.
1125+
import spack.hooks.sbang as sbang
1126+
orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(old_spack_prefix)
1127+
new_sbang = sbang.sbang_shebang_line()
1128+
prefix_to_prefix_text[orig_sbang] = new_sbang
11131129

11141130
tty.debug("Relocating package from",
11151131
"%s to %s." % (old_layout_root, new_layout_root))
@@ -1137,15 +1153,14 @@ def is_backup_file(file):
11371153
relocate.relocate_macho_binaries(files_to_relocate,
11381154
old_layout_root,
11391155
new_layout_root,
1140-
prefix_to_prefix, rel,
1156+
prefix_to_prefix_bin, rel,
11411157
old_prefix,
11421158
new_prefix)
1143-
11441159
if 'elf' in platform.binary_formats:
11451160
relocate.relocate_elf_binaries(files_to_relocate,
11461161
old_layout_root,
11471162
new_layout_root,
1148-
prefix_to_prefix, rel,
1163+
prefix_to_prefix_bin, rel,
11491164
old_prefix,
11501165
new_prefix)
11511166
# Relocate links to the new install prefix
@@ -1156,12 +1171,7 @@ def is_backup_file(file):
11561171

11571172
# For all buildcaches
11581173
# relocate the install prefixes in text files including dependencies
1159-
relocate.relocate_text(text_names,
1160-
old_layout_root, new_layout_root,
1161-
old_prefix, new_prefix,
1162-
old_spack_prefix,
1163-
new_spack_prefix,
1164-
prefix_to_prefix)
1174+
relocate.relocate_text(text_names, prefix_to_prefix_text)
11651175

11661176
paths_to_relocate = [old_prefix, old_layout_root]
11671177
paths_to_relocate.extend(prefix_to_hash.keys())
@@ -1171,22 +1181,13 @@ def is_backup_file(file):
11711181
map(lambda filename: os.path.join(workdir, filename),
11721182
buildinfo['relocate_binaries'])))
11731183
# relocate the install prefixes in binary files including dependencies
1174-
relocate.relocate_text_bin(files_to_relocate,
1175-
old_prefix, new_prefix,
1176-
old_spack_prefix,
1177-
new_spack_prefix,
1178-
prefix_to_prefix)
1179-
1180-
# If we are installing back to the same location
1181-
# relocate the sbang location if the spack directory changed
1184+
relocate.relocate_text_bin(files_to_relocate, prefix_to_prefix_bin)
1185+
1186+
# If we are installing back to the same location
1187+
# relocate the sbang location if the spack directory changed
11821188
else:
11831189
if old_spack_prefix != new_spack_prefix:
1184-
relocate.relocate_text(text_names,
1185-
old_layout_root, new_layout_root,
1186-
old_prefix, new_prefix,
1187-
old_spack_prefix,
1188-
new_spack_prefix,
1189-
prefix_to_prefix)
1190+
relocate.relocate_text(text_names, prefix_to_prefix_text)
11901191

11911192

11921193
def extract_tarball(spec, filename, allow_root=False, unsigned=False,

‎lib/spack/spack/filesystem_view.py‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import re
99
import shutil
1010
import sys
11+
from ordereddict_backport import OrderedDict
1112

1213
from llnl.util.link_tree import LinkTree, MergeConflictError
1314
from llnl.util import tty
@@ -65,32 +66,35 @@ def view_copy(src, dst, view, spec=None):
6566
# Not metadata, we have to relocate it
6667

6768
# Get information on where to relocate from/to
68-
prefix_to_projection = dict(
69-
(dep.prefix, view.get_projection_for_spec(dep))
70-
for dep in spec.traverse()
71-
)
69+
70+
# This is vestigial code for the *old* location of sbang. Previously,
71+
# sbang was a bash script, and it lived in the spack prefix. It is
72+
# now a POSIX script that lives in the install prefix. Old packages
73+
# will have the old sbang location in their shebangs.
74+
# TODO: Not sure which one to use...
75+
import spack.hooks.sbang as sbang
76+
orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root)
77+
new_sbang = sbang.sbang_shebang_line()
78+
79+
prefix_to_projection = OrderedDict({
80+
spec.prefix: view.get_projection_for_spec(spec),
81+
spack.paths.spack_root: view._root})
82+
83+
for dep in spec.traverse():
84+
prefix_to_projection[dep.prefix] = \
85+
view.get_projection_for_spec(dep)
7286

7387
if spack.relocate.is_binary(dst):
74-
# relocate binaries
7588
spack.relocate.relocate_text_bin(
7689
binaries=[dst],
77-
orig_install_prefix=spec.prefix,
78-
new_install_prefix=view.get_projection_for_spec(spec),
79-
orig_spack=spack.paths.spack_root,
80-
new_spack=view._root,
81-
new_prefixes=prefix_to_projection
90+
prefixes=prefix_to_projection
8291
)
8392
else:
84-
# relocate text
93+
prefix_to_projection[spack.store.layout.root] = view._root
94+
prefix_to_projection[orig_sbang] = new_sbang
8595
spack.relocate.relocate_text(
8696
files=[dst],
87-
orig_layout_root=spack.store.layout.root,
88-
new_layout_root=view._root,
89-
orig_install_prefix=spec.prefix,
90-
new_install_prefix=view.get_projection_for_spec(spec),
91-
orig_spack=spack.paths.spack_root,
92-
new_spack=view._root,
93-
new_prefixes=prefix_to_projection
97+
prefixes=prefix_to_projection
9498
)
9599

96100

0 commit comments

Comments
 (0)
X Tutup