|
8 | 8 | NamedTuple, |
9 | 9 | Optional, |
10 | 10 | Set, |
| 11 | + Tuple, |
11 | 12 | Type, |
12 | 13 | ) |
13 | 14 |
|
@@ -79,11 +80,17 @@ def _process_packages_v1( |
79 | 80 | elif version_url.scheme == 'file': |
80 | 81 | source = LocalSource(path=version_url.path) |
81 | 82 | else: |
82 | | - integrity = Integrity.parse(info['integrity']) |
| 83 | + integrity = None |
| 84 | + if 'integrity' in info: |
| 85 | + integrity = Integrity.parse(info['integrity']) |
| 86 | + |
83 | 87 | if 'resolved' in info: |
84 | | - source = ResolvedSource( |
85 | | - resolved=info['resolved'], integrity=integrity |
86 | | - ) |
| 88 | + if info['resolved'].startswith('git+'): |
| 89 | + source = self.parse_git_source(info['resolved']) |
| 90 | + else: |
| 91 | + source = ResolvedSource( |
| 92 | + resolved=info['resolved'], integrity=integrity |
| 93 | + ) |
87 | 94 | elif version_url.scheme in {'http', 'https'}: |
88 | 95 | source = PackageURLSource(resolved=version, integrity=integrity) |
89 | 96 | else: |
@@ -127,9 +134,8 @@ def _process_packages_v2( |
127 | 134 | elif resolved_url.scheme.startswith('git+'): |
128 | 135 | source = self.parse_git_source(info['resolved']) |
129 | 136 | else: |
130 | | - raise NotImplementedError( |
131 | | - f"Don't know how to handle package {install_path} in {lockfile}" |
132 | | - ) |
| 137 | + source = LocalSource(path=install_path) |
| 138 | + name = install_path |
133 | 139 |
|
134 | 140 | # NOTE We can't reliably determine the package name from the lockfile v2 syntax, |
135 | 141 | # but we need it for registry queries and special source processing; |
@@ -200,7 +206,7 @@ def __init__( |
200 | 206 | self.all_lockfiles: Set[Path] = set() |
201 | 207 | # Mapping of lockfiles to a dict of the Git source target paths and GitSource objects. |
202 | 208 | self.git_sources: DefaultDict[ |
203 | | - Path, Dict[Path, (str, GitSource)] |
| 209 | + Path, Dict[Path, Tuple[str, GitSource]] |
204 | 210 | ] = collections.defaultdict(lambda: {}) |
205 | 211 | # FIXME better pass the same provider object we created in main |
206 | 212 | self.rcfile_provider = NpmRCFileProvider() |
@@ -364,6 +370,46 @@ async def generate_package(self, package: Package) -> None: |
364 | 370 | self.git_sources[package.lockfile][path] = (package.name, source) |
365 | 371 | self.gen.add_git_source(source.url, source.commit, path) |
366 | 372 |
|
| 373 | + url = urllib.parse.urlparse(source.url) |
| 374 | + if url. netloc == '[email protected]' or url. netloc == 'github.com': |
| 375 | + url = url._replace( |
| 376 | + netloc='codeload.github.com', path=re.sub('.git$', '', url.path) |
| 377 | + ) |
| 378 | + tarball_url = url._replace( |
| 379 | + path=re.sub('$', f'/tar.gz/{source.commit}', url.path) |
| 380 | + ) |
| 381 | + index_url = tarball_url.geturl() |
| 382 | + elif url. netloc == '[email protected]' or url. netloc == 'gitlab.com': |
| 383 | + url = url._replace( |
| 384 | + netloc='gitlab.com', path=re.sub('.git$', '', url.path) |
| 385 | + ) |
| 386 | + tarball_url = url._replace( |
| 387 | + path=re.sub( |
| 388 | + '$', |
| 389 | + f'/-/archive/{source.commit}/{package.name}-{source.commit}.tar.gz', |
| 390 | + url.path, |
| 391 | + ) |
| 392 | + ) |
| 393 | + index_url = url._replace( |
| 394 | + path=re.sub( |
| 395 | + '$', f'/repository/archive.tar.gz?ref={source.commit}', url.path |
| 396 | + ) |
| 397 | + ).geturl() |
| 398 | + metadata = await RemoteUrlMetadata.get( |
| 399 | + tarball_url.geturl(), cachable=True, integrity_algorithm='sha512' |
| 400 | + ) |
| 401 | + |
| 402 | + self.gen.add_url_source( |
| 403 | + url=tarball_url.geturl(), |
| 404 | + integrity=metadata.integrity, |
| 405 | + destination=self.get_cacache_content_path(metadata.integrity), |
| 406 | + ) |
| 407 | + |
| 408 | + self.add_index_entry( |
| 409 | + url=index_url, |
| 410 | + metadata=metadata, |
| 411 | + ) |
| 412 | + |
367 | 413 | elif isinstance(source, LocalSource): |
368 | 414 | pass |
369 | 415 |
|
@@ -417,61 +463,91 @@ def _finalize(self) -> None: |
417 | 463 | ) |
418 | 464 |
|
419 | 465 | if self.git_sources: |
420 | | - # Generate jq script to patch the package.json file |
421 | | - script = r""" |
422 | | - walk( |
423 | | - if type == "object" |
424 | | - then |
425 | | - to_entries | map( |
426 | | - if (.key | type == "string") and $data[.key] |
427 | | - then .value = "git+file://\($buildroot)/\($data[.key])" |
428 | | - else . |
429 | | - end |
430 | | - ) | from_entries |
431 | | - else . |
432 | | - end |
433 | | - ) |
434 | | - """ |
| 466 | + # Generate jq scripts to patch the package*.json files. |
| 467 | + scripts = { |
| 468 | + 'package.json': r""" |
| 469 | + walk( |
| 470 | + if type == "object" |
| 471 | + then |
| 472 | + to_entries | map( |
| 473 | + if (.key | type == "string") and $data[.key] |
| 474 | + then .value = "git+file://\($buildroot)/\($data[.key])" |
| 475 | + else . |
| 476 | + end |
| 477 | + ) | from_entries |
| 478 | + else . |
| 479 | + end |
| 480 | + ) |
| 481 | + """, |
| 482 | + 'package-lock.json': r""" |
| 483 | + walk( |
| 484 | + if type == "object" and (.version | type == "string") and $data[.version] |
| 485 | + then |
| 486 | + .resolved = "git+file:\($buildroot)/\($data[.version])" |
| 487 | + else . |
| 488 | + end |
| 489 | + ) |
| 490 | + """, |
| 491 | + } |
435 | 492 |
|
436 | 493 | for lockfile, sources in self.git_sources.items(): |
437 | 494 | prefix = self.relative_lockfile_dir(lockfile) |
438 | | - data: Dict[str, str] = {} |
439 | | - |
440 | | - for path, name_source in sources.items(): |
441 | | - name, source = name_source |
442 | | - new_version = f'{path}#{source.commit}' |
443 | | - data[name] = new_version |
444 | | - |
445 | | - filename = 'package.json' |
446 | | - target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename |
447 | | - script = textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '') |
448 | | - json_data = json.dumps(data) |
449 | | - patch_commands[lockfile].append( |
450 | | - 'jq' |
451 | | - ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"' |
452 | | - f' --argjson data {shlex.quote(json_data)}' |
453 | | - f' {shlex.quote(script)} {target}' |
454 | | - f' > {target}.new' |
455 | | - ) |
456 | | - patch_commands[lockfile].append(f'mv {target}{{.new,}}') |
| 495 | + data: Dict[str, Dict[str, str]] = { |
| 496 | + 'package.json': {}, |
| 497 | + 'package-lock.json': {}, |
| 498 | + } |
| 499 | + |
| 500 | + with open(lockfile) as fp: |
| 501 | + lockfile_v1 = json.load(fp)['lockfileVersion'] == 1 |
| 502 | + |
| 503 | + if lockfile_v1: |
| 504 | + for path, name_source in sources.items(): |
| 505 | + GIT_URL_PREFIX = 'git+' |
| 506 | + name, source = name_source |
| 507 | + new_version = f'{path}#{source.commit}' |
| 508 | + data['package.json'][name] = new_version |
| 509 | + data['package-lock.json'][source.original] = new_version |
| 510 | + |
| 511 | + if source.original.startswith(GIT_URL_PREFIX): |
| 512 | + data['package-lock.json'][ |
| 513 | + source.original[len(GIT_URL_PREFIX) :] |
| 514 | + ] = new_version |
| 515 | + |
| 516 | + for filename, script in scripts.items(): |
| 517 | + target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename |
| 518 | + script = ( |
| 519 | + textwrap.dedent(script.lstrip('\n')) |
| 520 | + .strip() |
| 521 | + .replace('\n', '') |
| 522 | + ) |
| 523 | + json_data = json.dumps(data[filename]) |
| 524 | + patch_commands[lockfile].append( |
| 525 | + 'jq' |
| 526 | + ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"' |
| 527 | + f' --argjson data {shlex.quote(json_data)}' |
| 528 | + f' {shlex.quote(script)} {target}' |
| 529 | + f' > {target}.new' |
| 530 | + ) |
| 531 | + patch_commands[lockfile].append(f'mv {target}{{.new,}}') |
457 | 532 |
|
458 | | - patch_all_commands: List[str] = [] |
459 | | - for lockfile in self.all_lockfiles: |
460 | | - patch_dest = ( |
461 | | - self.gen.data_root / 'patch' / self.relative_lockfile_dir(lockfile) |
462 | | - ) |
463 | | - # Don't use with_extension to avoid problems if the package has a . in its name. |
464 | | - patch_dest = patch_dest.with_name(patch_dest.name + '.sh') |
| 533 | + if len(patch_commands) > 0: |
| 534 | + patch_all_commands: List[str] = [] |
| 535 | + for lockfile in self.all_lockfiles: |
| 536 | + patch_dest = ( |
| 537 | + self.gen.data_root / 'patch' / self.relative_lockfile_dir(lockfile) |
| 538 | + ) |
| 539 | + # Don't use with_extension to avoid problems if the package has a . in its name. |
| 540 | + patch_dest = patch_dest.with_name(patch_dest.name + '.sh') |
465 | 541 |
|
466 | | - self.gen.add_script_source(patch_commands[lockfile], patch_dest) |
467 | | - patch_all_commands.append(f'$FLATPAK_BUILDER_BUILDDIR/{patch_dest}') |
| 542 | + self.gen.add_script_source(patch_commands[lockfile], patch_dest) |
| 543 | + patch_all_commands.append(f'$FLATPAK_BUILDER_BUILDDIR/{patch_dest}') |
468 | 544 |
|
469 | | - patch_all_dest = self.gen.data_root / 'patch-all.sh' |
470 | | - self.gen.add_script_source(patch_all_commands, patch_all_dest) |
| 545 | + patch_all_dest = self.gen.data_root / 'patch-all.sh' |
| 546 | + self.gen.add_script_source(patch_all_commands, patch_all_dest) |
471 | 547 |
|
472 | | - if not self.no_autopatch: |
473 | | - # FLATPAK_BUILDER_BUILDDIR isn't defined yet for script sources. |
474 | | - self.gen.add_command(f'FLATPAK_BUILDER_BUILDDIR=$PWD {patch_all_dest}') |
| 548 | + if not self.no_autopatch: |
| 549 | + # FLATPAK_BUILDER_BUILDDIR isn't defined yet for script sources. |
| 550 | + self.gen.add_command(f'FLATPAK_BUILDER_BUILDDIR=$PWD {patch_all_dest}') |
475 | 551 |
|
476 | 552 | if self.index_entries: |
477 | 553 | for path, entry in self.index_entries.items(): |
|
0 commit comments