From 4f01784899f7b68d56b6d4039e30b504e100e9b6 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 23 May 2024 12:56:28 -0700 Subject: [PATCH 1/9] Update data structure and logging, add cocoapods support #365 - Also replaced PURL normalization option with default deduplication. Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 467 ++--- .../purlcli/expected_metadata_output.json | 1310 ++++++------ .../expected_metadata_output_superseded.json | 588 ------ .../expected_metadata_output_unique.json | 353 ---- .../data/purlcli/expected_urls_output.json | 59 +- .../purlcli/expected_urls_output_head.json | 83 +- .../expected_urls_output_head_mock.json | 8 - .../purlcli/expected_urls_output_unique.json | 240 --- .../purlcli/expected_validate_output.json | 11 - .../expected_validate_output_unique.json | 77 - .../purlcli/expected_versions_output.json | 231 +- .../expected_versions_output_unique.json | 140 -- purldb-toolkit/tests/test_purlcli.py | 735 +++---- purldb-toolkit/tests/test_purlcli_live.py | 1864 ++++------------- 14 files changed, 1667 insertions(+), 4499 deletions(-) delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 0269ce97..917c1008 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -17,11 +17,13 @@ import click import requests from fetchcode.package import info -from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS, versions +from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS +from fetchcode.package_versions import versions from packageurl import PackageURL from packageurl.contrib import purl2url LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") +logger = logging.getLogger(__name__) @click.group() @@ -52,162 +54,153 @@ def purlcli(): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_metadata(purls, output, file, unique): +def get_metadata(purls, output, file): """ Given one or more PURLs, for each PURL, return a mapping of metadata fetched from the fetchcode package.py info() function. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - metadata_info = get_metadata_details(purls, output, file, unique, command_name) + metadata_info = get_metadata_details(purls, output, file, command_name) json.dump(metadata_info, output, indent=4) -def get_metadata_details(purls, output, file, unique, command_name): +def get_metadata_details(purls, output, file, command_name): """ Return a dictionary containing metadata for each PURL in the `purls` input - list. `check_metadata_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ metadata_details = {} metadata_details["headers"] = [] metadata_details["packages"] = [] - metadata_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - - purl_data = {} - purl_data["purl"] = purl - - metadata_purl = check_metadata_purl(purl) - - if command_name == "metadata" and metadata_purl: - metadata_warnings[purl] = metadata_purl + metadata_purl_status = check_metadata_purl(purl) + if command_name == "metadata" and metadata_purl_status in [ + "validation_error", + "not_valid", + "valid_but_not_supported", + "not_in_upstream_repo", + ]: + metadata_warnings[purl] = metadata_purl_status continue - + if command_name == "metadata" and metadata_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + metadata_warnings[purl] = metadata_purl_status metadata_collection = collect_metadata(purl) - purl_data["metadata"] = metadata_collection - metadata_details["packages"].append(purl_data) + metadata_details["packages"].extend(metadata_collection) + + print(f"\nmetadata_warnings = {metadata_warnings}") metadata_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=metadata_warnings, ) - return metadata_details def collect_metadata(purl): """ - Return a list of release-based metadata collections from fetchcode/package.py. + Return a list of release-based metadata collections (a fetchcode Package) + from fetchcode/package.py. """ collected_metadata = [] for release in list(info(purl)): + if release is None: + continue release_detail = release.to_dict() release_detail.move_to_end("purl", last=False) collected_metadata.append(release_detail) - return collected_metadata def check_metadata_purl(purl): """ - Return a variable identifying the message for printing to the console by - get_metadata_details() if (1) the input PURL is invalid, (2) its type is not - supported by `metadata` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `metadata` command. + Return a variable identifying the warning if (1) the input PURL is invalid, + (2) its type is not supported by `metadata` or (3) its existence was not + validated (e.g., "does not exist in the upstream repo"). This will be + reported by construct_headers() in the `warnings` field of the `header` + section of the JSON object returned by the `metadata` command. """ check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" # This is manually constructed from a visual inspection of fetchcode/package.py. metadata_supported_ecosystems = [ "bitbucket", "cargo", + "generic", "github", + "gnu", "npm", + "openssl", "pypi", "rubygems", + # NOTE: cocoapods support added subject to fetchcode/package.py PR approval and new release. + "cocoapods", ] metadata_purl = PackageURL.from_string(purl) if metadata_purl.type not in metadata_supported_ecosystems: return "valid_but_not_supported" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" -def normalize_purls(purls, unique): - """ - If the command includes the `--unique` flag, take the list of input PURLs, - remove the portion of the PURL that starts with a PURL separator (`@`, `?` - or `#`), and return a deduplicated list of the resulting PURLs (in - `input_purls`) and a list of tuples of each pair of the original input PURL - and the normalized PURL (in `normalized_purls`). - """ - input_purls = [] - normalized_purls = [] - if unique: - for purl in purls: - input_purl = purl - purl = purl.strip() - purl = re.split("[@,?,#,]+", purl)[0] - normalized_purl = purl - normalized_purls.append((input_purl, normalized_purl)) - if normalized_purl not in input_purls: - input_purls.append(normalized_purl) - else: - input_purls = purls +def deduplicate_purls(purls): + """ + Deduplicate all input PURLs. PURLs with different versions or no version + are treated as unique for purposes of deduplication. + """ + reviewed = set() + deduplicated_purls = [] + duplicate_purls = [] + for purl in purls: + purl = purl.strip() + if purl not in reviewed: + reviewed.add(purl) + deduplicated_purls.append(purl) + else: + duplicate_purls.append(purl) + return deduplicated_purls, duplicate_purls - return input_purls, normalized_purls + +def read_log_file(log_file_path): + log_file = log_file_path + if log_file.is_file(): + with open(log_file_path, "r") as log_file: + return log_file.readlines() + else: + return [] def construct_headers( - purls=None, + deduplicated_purls=None, + duplicate_purls=None, output=None, file=None, command_name=None, head=None, - normalized_purls=None, - unique=None, purl_warnings=None, ): """ @@ -219,7 +212,7 @@ def construct_headers( errors = [] warnings = [] - context_purls = [p for p in purls] + context_purls = [p for p in deduplicated_purls] context_file = file context_file_name = None if context_file: @@ -235,28 +228,20 @@ def construct_headers( if head: options["--head"] = True - if unique: - options["--unique"] = True - if isinstance(output, str): options["--output"] = output else: options["--output"] = output.name headers_content["options"] = options - headers_content["purls"] = purls - - if (command_name in ["metadata", "urls", "validate", "versions"]) and unique: - for input_purl, normalized_purl in normalized_purls: - if input_purl != normalized_purl: - warnings.append( - f"input PURL: '{input_purl}' normalized to '{normalized_purl}'" - ) + if command_name in ["metadata", "urls", "validate", "versions"]: + if duplicate_purls: + for duplicate in duplicate_purls: + logger.warning(f"Duplicate input PURL removed: {duplicate}") - for purl in purls: + for purl in deduplicated_purls: if not purl: continue - warning_text = { "error_fetching_purl": f"'error fetching {purl}'", "validation_error": f"'{purl}' encountered a validation error", @@ -266,25 +251,25 @@ def construct_headers( "not_in_upstream_repo": f"'{purl}' does not exist in the upstream repo", "check_existence_not_supported": f"'check_existence' is not supported for '{purl}'", } - if command_name in ["metadata", "urls", "validate", "versions"]: purl_warning = purl_warnings.get(purl, None) - if purl_warning: warning = warning_text[purl_warning] - warnings.append(warning) + logger.warning(warning) continue log_file = Path(LOG_FILE_LOCATION) - if log_file.is_file(): - with open(log_file, "r") as f: - for line in f: - errors.append(line) + log_file_contents = read_log_file(log_file) + if log_file_contents: + for line in log_file_contents: + if line.startswith("ERROR"): + errors.append(line[8:-1]) + elif line.startswith("WARNING"): + warnings.append(line[10:-1]) headers_content["errors"] = errors headers_content["warnings"] = warnings headers.append(headers_content) - return headers @@ -309,102 +294,81 @@ def construct_headers( required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) @click.option( "--head", is_flag=True, required=False, help="Validate each URL's existence with a head request.", ) -def get_urls(purls, output, file, unique, head): +def get_urls(purls, output, file, head): """ Given one or more PURLs, for each PURL, return a list of all known URLs fetched from the packageurl-python purl2url.py code. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - urls_info = get_urls_details(purls, output, file, unique, head, command_name) + urls_info = get_urls_details(purls, output, file, head, command_name) json.dump(urls_info, output, indent=4) -def get_urls_details(purls, output, file, unique, head, command_name): +def get_urls_details(purls, output, file, head, command_name): """ Return a dictionary containing URLs for each PURL in the `purls` input - list. `check_urls_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ urls_details = {} urls_details["headers"] = [] urls_details["packages"] = [] - urls_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: url_detail = {} url_detail["purl"] = purl - purl = purl.strip() if not purl: continue - - purl_status = check_urls_purl(purl) - - if command_name == "urls" and purl_status in [ + urls_purl_status = check_urls_purl(purl) + if command_name == "urls" and urls_purl_status in [ "validation_error", "not_valid", "valid_but_not_supported", "not_in_upstream_repo", ]: - urls_warnings[purl] = purl_status + urls_warnings[purl] = urls_purl_status continue + if command_name == "urls" and urls_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + urls_warnings[purl] = urls_purl_status - if command_name == "urls" and purl_status in ["valid_but_not_fully_supported"]: - urls_warnings[purl] = purl_status - - # Add the URLs. url_purl = PackageURL.from_string(purl) url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} - url_detail["inferred_urls"] = [ {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} - - url_detail["repo_download_url_by_package_type"] = { - "url": purl2url.get_repo_download_url_by_package_type( - url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - ) - } - + url_detail["repo_download_url_by_package_type"] = {"url": None} + if url_purl.version: + url_detail["repo_download_url_by_package_type"] = { + "url": purl2url.get_repo_download_url_by_package_type( + url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + ) + } url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} - url_detail["url"] = {"url": purl2url.get_url(purl)} - - # Add the http status code data. url_list = [ "download_url", # "inferred_urls" has to be handled separately because it has a nested list "repo_download_url", "repo_download_url_by_package_type", "repo_url", - "url", ] if head: for purlcli_url in url_list: @@ -414,7 +378,6 @@ def get_urls_details(purls, output, file, unique, head, command_name): url_detail[purlcli_url]["head_request_status_code"] = make_head_request( url_detail[purlcli_url]["url"] ).get("head_request") - for inferred_url in url_detail["inferred_urls"]: inferred_url["get_request_status_code"] = make_head_request( inferred_url["url"] @@ -422,44 +385,34 @@ def get_urls_details(purls, output, file, unique, head, command_name): inferred_url["head_request_status_code"] = make_head_request( inferred_url["url"] ).get("head_request") - urls_details["packages"].append(url_detail) urls_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, head=head, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=urls_warnings, ) - return urls_details def make_head_request(url_detail): """ - Make a head request (and as noted below, a get request as well, at least - for now) and return a dictionary containing status code data for the - incoming PURL URL. - - For now, this returns both get and head request status code data so the - user can evaluate -- requests.get() and requests.head() sometimes return - different status codes and sometimes return inaccurate codes, e.g., a - 404 when the URL actually exists. + Make a head request and get request and return a dictionary containing + status code data for the incoming PURL URL. This returns both get and + head request status code data so the user can evaluate -- requests.get() + and requests.head() sometimes return different status codes and sometimes + return inaccurate codes, e.g., a 404 when the URL actually exists. """ if url_detail is None: return {"get_request": "N/A", "head_request": "N/A"} - get_response = requests.get(url_detail) get_request_status_code = get_response.status_code - head_response = requests.head(url_detail) head_request_status_code = head_response.status_code - - # Return a dictionary for readability. return { "get_request": get_request_status_code, "head_request": head_request_status_code, @@ -476,17 +429,17 @@ def check_urls_purl(purl): if check_validation is None: return "validation_error" results = check_validation - if results["valid"] == False: return "not_valid" # Both of these lists are manually constructed from a visual inspection of # packageurl-python/src/packageurl/contrib/purl2url.py. - - # This list applies to the purl2url.py `repo_url` section: + # This list applies to the purl2url.py `repo_url` section: urls_supported_ecosystems_repo_url = [ "bitbucket", "cargo", + # NOTE: Temp for cocoapods dev work in purl2url. Keep in the list uncommented -- I still need to respond to Tushar's comments but the real code work is done, now supported for repo_url. + "cocoapods", "gem", "github", "gitlab", @@ -497,7 +450,6 @@ def check_urls_purl(purl): "pypi", "rubygems", ] - # This list applies to the purl2url.py `download_url` section: urls_supported_ecosystems_download_url = [ "bitbucket", @@ -510,7 +462,6 @@ def check_urls_purl(purl): "nuget", "rubygems", ] - urls_purl = PackageURL.from_string(purl) if ( @@ -518,10 +469,8 @@ def check_urls_purl(purl): and urls_purl.type not in urls_supported_ecosystems_download_url ): return "valid_but_not_supported" - if results["exists"] == False: return "not_in_upstream_repo" - if ( urls_purl.type in urls_supported_ecosystems_repo_url and urls_purl.type not in urls_supported_ecosystems_download_url @@ -529,8 +478,9 @@ def check_urls_purl(purl): urls_purl.type not in urls_supported_ecosystems_repo_url and urls_purl.type in urls_supported_ecosystems_download_url ): - return "valid_but_not_fully_supported" + if results["exists"] == None: + return "check_existence_not_supported" @purlcli.command(name="validate") @@ -554,73 +504,55 @@ def check_urls_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def validate(purls, output, file, unique): +def validate(purls, output, file): """ Check the syntax and upstream repo status of one or more PURLs. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - validated_purls = get_validate_details(purls, output, file, unique, command_name) + validated_purls = get_validate_details(purls, output, file, command_name) json.dump(validated_purls, output, indent=4) -def get_validate_details(purls, output, file, unique, command_name): +def get_validate_details(purls, output, file, command_name): """ Return a dictionary containing validation data for each PURL in the `purls` input list. """ validate_details = {} validate_details["headers"] = [] - validate_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) validate_details["packages"] = [] - clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - - validated_purl = check_validate_purl(purl) - - if command_name == "validate" and validated_purl in [ + validated_purl_status = check_validate_purl(purl) + if command_name == "validate" and validated_purl_status in [ "validation_error", "not_valid", "valid_but_not_supported", "not_in_upstream_repo", "check_existence_not_supported", ]: - validate_warnings[purl] = validated_purl - - if validated_purl: + validate_warnings[purl] = validated_purl_status + if validated_purl_status: validate_details["packages"].append(validate_purl(purl)) validate_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=validate_warnings, ) - return validate_details @@ -632,18 +564,13 @@ def check_validate_purl(purl): check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == True: + elif check_validation["exists"] == True: return check_validation - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" @@ -668,33 +595,27 @@ def validate_purl(purl): "nuget", "pypi", """ - logger = logging.getLogger(__name__) + logging.basicConfig( + level=logging.WARN, + format="%(levelname)s - %(message)s", + filename=LOG_FILE_LOCATION, + filemode="w", + ) api_query = "https://public.purldb.io/api/validate/" request_body = {"purl": purl, "check_existence": True} try: response = requests.get(api_query, params=request_body).json() - except json.decoder.JSONDecodeError as e: - - print(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - - logging.basicConfig( - filename=LOG_FILE_LOCATION, - level=logging.ERROR, - format="%(levelname)s - %(message)s", - filemode="w", - ) - logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - except Exception as e: - print(f"'validate' endpoint error for '{purl}': {e}") - + logger.error(f"'validate' endpoint error for '{purl}': {e}") else: if response is None: - print(f"'{purl}' -- response.status_code for None = {response.status_code}") + logger.error( + f"'{purl}' -- response.status_code for None = {response.status_code}" + ) return response @@ -719,99 +640,89 @@ def validate_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_versions(purls, output, file, unique): +def get_versions(purls, output, file): """ Given one or more PURLs, return a list of all known versions for each PURL. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - purl_versions = get_versions_details(purls, output, file, unique, command_name) + purl_versions = get_versions_details(purls, output, file, command_name) json.dump(purl_versions, output, indent=4) -def get_versions_details(purls, output, file, unique, command_name): +def get_versions_details(purls, output, file, command_name): """ Return a list of dictionaries containing version-related data for each PURL - in the `purls` input list. `check_versions_purl()` will print an error - message to the console (also displayed in the JSON output) when necessary. + in the `purls` input list. """ versions_details = {} versions_details["headers"] = [] versions_details["packages"] = [] - versions_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - purl_data = {} purl_data["purl"] = purl - - versions_purl = check_versions_purl(purl) - - if command_name == "versions" and versions_purl: - versions_warnings[purl] = versions_purl + versions_purl_status = check_versions_purl(purl) + if command_name == "versions" and versions_purl_status in [ + "validation_error", + "not_valid", + "valid_but_not_supported", + "not_in_upstream_repo", + ]: + versions_warnings[purl] = versions_purl_status continue - + if command_name == "versions" and versions_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + versions_warnings[purl] = versions_purl_status version_collection = collect_versions(purl) - - purl_data["versions"] = version_collection - versions_details["packages"].append(purl_data) + versions_details["packages"].extend(version_collection) versions_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=versions_warnings, ) - return versions_details def collect_versions(purl): """ Return a list of version objects collected from fetchcode/package_versions.py. + + We use `versions()` from fetchcode/package_versions.py, which keeps the + version (if any) of the input PURL in its output, so + "pkg:pypi/fetchcode@0.3.0" is returned as "pkg:pypi/fetchcode@0.3.0@0.1.0", + "pkg:pypi/fetchcode@0.3.0@0.2.0" etc. Thus, we remove any string starting + with `@` first. """ collected_versions = [] for package_version in list(versions(purl)): purl_version_data = {} purl_version = package_version.value - # We use `versions()` from fetchcode/package_versions.py, which - # keeps the version (if any) of the input PURL in its output, so - # "pkg:pypi/fetchcode@0.3.0" is returned as - # "pkg:pypi/fetchcode@0.3.0@0.1.0", "pkg:pypi/fetchcode@0.3.0@0.2.0" - # etc. Thus, we remove any string starting with `@` first. - raw_purl = purl = re.split("[@,]+", purl)[0] + raw_purl = re.split("[@,]+", purl)[0] nested_purl = raw_purl + "@" + f"{purl_version}" + pkg_ver_release_date = package_version.release_date + pkg_ver_release_date_no_time = pkg_ver_release_date.date() purl_version_data["purl"] = nested_purl purl_version_data["version"] = f"{purl_version}" - purl_version_data["release_date"] = f"{package_version.release_date}" - + purl_version_data["release_date"] = f"{pkg_ver_release_date_no_time}" collected_versions.append(purl_version_data) - return collected_versions @@ -820,11 +731,9 @@ def check_versions_purl(purl): Return a variable identifying the message for printing to the console by get_versions_details() if (1) the input PURL is invalid, (2) its type is not supported by `versions` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `versions` command. + "does not exist in the upstream repo"). This message will also be reported + by construct_headers() in the `warnings` field of the `header` section of + the JSON object returned by the `versions` command. Note for dev purposes: SUPPORTED_ECOSYSTEMS (imported from fetchcode.package_versions) comprises the following types: @@ -846,43 +755,35 @@ def check_versions_purl(purl): check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" supported = SUPPORTED_ECOSYSTEMS versions_purl = PackageURL.from_string(purl) - if versions_purl.type not in supported: return "valid_but_not_supported" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" - # This handles the conflict between the `validate`` endpoint (treats # both "pkg:deb/debian/2ping" and "pkg:deb/2ping" as valid) and # fetchcode.package_versions versions() (returns None for "pkg:deb/2ping"). - if versions(purl) is None: + elif versions(purl) is None: return "valid_but_not_supported" def check_for_duplicate_input_sources(purls, file): if purls and file: raise click.UsageError("Use either purls or file but not both.") - - if not (purls or file): + elif not (purls or file): raise click.UsageError("Use either purls or file.") def clear_log_file(): log_file = Path(LOG_FILE_LOCATION) - - if log_file.is_file(): - os.remove(log_file) + with open(log_file, "w"): + pass if __name__ == "__main__": diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 1f5cb267..32df4492 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -19,17 +19,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", @@ -43,680 +32,655 @@ "packages": [ { "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": { + "os": "windows" + }, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "cargo", + "namespace": null, + "name": "banquo", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "api_url": "https://crates.io/api/v1/crates/banquo", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:cargo/banquo@0.1.0", + "type": "cargo", + "namespace": null, + "name": "banquo", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", + "api_url": "https://crates.io/api/v1/crates/banquo", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "BSD-3-Clause", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "rubygems", + "namespace": null, + "name": "rails", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://rubyonrails.org", + "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", + "api_url": "https://rubygems.org/api/v1/gems/rails.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/rails/rails/issues", + "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": [ + "MIT" + ], + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json deleted file mode 100644 index a237dae1..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json +++ /dev/null @@ -1,588 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json deleted file mode 100644 index 5ede4644..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json +++ /dev/null @@ -1,353 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index dc1d1e4a..dac9392b 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -30,28 +30,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command", @@ -66,6 +44,7 @@ "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", "'pkg:nginx/nginx' not supported with `urls` command", "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", "'pkg:pypi/matchcode' does not exist in the upstream repo", "'abcdefg' not valid", "'pkg/abc' not valid" @@ -91,9 +70,6 @@ }, "repo_url": { "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" } }, { @@ -114,9 +90,6 @@ }, "repo_url": { "url": "https://pypi.org/project/fetchcode/0.3.0/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" } }, { @@ -137,9 +110,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" } }, { @@ -160,9 +130,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -183,9 +150,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -206,9 +170,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -229,9 +190,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -252,9 +210,6 @@ }, "repo_url": { "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" } }, { @@ -275,9 +230,6 @@ }, "repo_url": { "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" } }, { @@ -298,9 +250,6 @@ }, "repo_url": { "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" } }, { @@ -321,9 +270,6 @@ }, "repo_url": { "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" } }, { @@ -347,9 +293,6 @@ }, "repo_url": { "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index d5b256e4..b256c973 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -31,28 +31,6 @@ "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command", @@ -67,6 +45,7 @@ "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", "'pkg:nginx/nginx' not supported with `urls` command", "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", "'pkg:pypi/matchcode' does not exist in the upstream repo", "'abcdefg' not valid", "'pkg/abc' not valid" @@ -102,11 +81,6 @@ "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -137,11 +111,6 @@ "url": "https://pypi.org/project/fetchcode/0.3.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -172,11 +141,6 @@ "url": "https://pypi.org/project/dejacode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -207,11 +171,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -242,11 +201,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -277,11 +231,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -312,11 +261,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -347,11 +291,6 @@ "url": "https://crates.io/crates/banquo", "get_request_status_code": 404, "head_request_status_code": 404 - }, - "url": { - "url": "https://crates.io/crates/banquo", - "get_request_status_code": 404, - "head_request_status_code": 404 } }, { @@ -382,11 +321,6 @@ "url": "https://crates.io/crates/socksprox", "get_request_status_code": 404, "head_request_status_code": 404 - }, - "url": { - "url": "https://crates.io/crates/socksprox", - "get_request_status_code": 404, - "head_request_status_code": 404 } }, { @@ -417,11 +351,6 @@ "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -452,11 +381,6 @@ "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -492,11 +416,6 @@ "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", "get_request_status_code": 200, "head_request_status_code": 404 - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", - "get_request_status_code": 200, - "head_request_status_code": 404 } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 5f6c2e21..16578130 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -12,9 +12,6 @@ "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command" @@ -50,11 +47,6 @@ "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json deleted file mode 100644 index 193b8096..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "urls", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0#how/are/you' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@10.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:nginx/nginx@0.8.9?os=windows' normalized to 'pkg:nginx/nginx'", - "input PURL: 'pkg:nuget/auth0-aspnet@1.1.0' normalized to 'pkg:nuget/auth0-aspnet'", - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" - } - }, - { - "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" - } - }, - { - "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" - } - }, - { - "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" - } - }, - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:nuget/auth0-aspnet", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json index ac85f7c7..d1f74eba 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json @@ -19,17 +19,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json deleted file mode 100644 index 280388f2..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "validate", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'check_existence' is not supported for 'pkg:nginx/nginx'", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode" - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:cargo/banquo" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - "valid": true - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/rails" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/rails", - "valid": true - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json index 93feeb96..e9c6abf8 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json @@ -18,16 +18,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", @@ -38,139 +28,114 @@ ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "version": "0.3.0", + "release_date": "2023-12-18" }, { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" }, { - "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18" + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18" + }, + { + "purl": "pkg:cargo/banquo@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07" + }, + { + "purl": "pkg:hex/coherence@0.8.0", + "version": "0.8.0", + "release_date": "2023-09-22" + }, + { + "purl": "pkg:hex/coherence@0.5.2", + "version": "0.5.2", + "release_date": "2018-09-03" + }, + { + "purl": "pkg:hex/coherence@0.5.1", + "version": "0.5.1", + "release_date": "2018-08-28" + }, + { + "purl": "pkg:hex/coherence@0.5.0", + "version": "0.5.0", + "release_date": "2017-08-02" + }, + { + "purl": "pkg:hex/coherence@0.4.0", + "version": "0.4.0", + "release_date": "2017-07-03" + }, + { + "purl": "pkg:hex/coherence@0.3.1", + "version": "0.3.1", + "release_date": "2016-11-27" + }, + { + "purl": "pkg:hex/coherence@0.3.0", + "version": "0.3.0", + "release_date": "2016-08-28" + }, + { + "purl": "pkg:hex/coherence@0.2.0", + "version": "0.2.0", + "release_date": "2016-07-30" + }, + { + "purl": "pkg:hex/coherence@0.1.3", + "version": "0.1.3", + "release_date": "2016-07-19" + }, + { + "purl": "pkg:hex/coherence@0.1.2", + "version": "0.1.2", + "release_date": "2016-07-12" + }, + { + "purl": "pkg:hex/coherence@0.1.1", + "version": "0.1.1", + "release_date": "2016-07-11" }, { "purl": "pkg:hex/coherence@0.1.0", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] + "version": "0.1.0", + "release_date": "2016-07-11" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json deleted file mode 100644 index 1991c21f..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:hex/coherence@0.1.0' normalized to 'pkg:hex/coherence'", - "'pkg:nginx/nginx' not supported with `versions` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] - }, - { - "purl": "pkg:hex/coherence", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index f9cc12d5..7a8c90de 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -10,15 +10,14 @@ import json import os from collections import OrderedDict -from contextlib import redirect_stdout -from io import StringIO from unittest import mock import pytest import requests from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -236,193 +235,184 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", None), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("download_url", None), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", ), - ], - } + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), ], } input_purls = ["pkg:pypi/fetchcode"] - output = "" file = "" command_name = "metadata" - unique = False purl_metadata_data = purlcli.get_metadata_details( input_purls, output, file, - unique, command_name, ) @@ -513,113 +503,40 @@ def test_check_metadata_purl_multiple(self, mock_validate_purl): purl_metadata = purlcli.check_metadata_purl(input_purl) assert purl_metadata == expected_state - @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", - [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), - ( - [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls + def test_deduplicate_purls(self): + input_purls = [ + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + "pkg:pypi/fetchcode@0.2.0", + ] + actual_output = purlcli.deduplicate_purls(input_purls) + expected_output = ( + ["pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0"], + [ + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + ], + ) + assert actual_output == expected_output @pytest.mark.parametrize( "test_input,expected", [ ( [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode@0.2.0", ], [ @@ -629,40 +546,51 @@ def test_normalize_purls( "--file": None, "--output": "", "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode@0.2.0", ], "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0", ], } ], ), ], ) - def test_construct_headers(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_deduplicate_purls_construct_headers( + self, mock_read_log_file, test_input, expected + ): + mock_read_log_file.return_value = [ + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0\n", + ] + metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, + purl_warnings={}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -690,43 +618,33 @@ def test_construct_headers(self, test_input, expected): "pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0", ], - "--unique": True, "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", + "'pkg:gem/bundler-sass' not supported with `metadata` command" ], } ], ), ], ) - def test_construct_headers_unique(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_construct_headers(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", + ] + metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), - ], - unique=True, purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -734,8 +652,9 @@ def test_construct_headers_unique(self, test_input, expected): class TestPURLCLI_urls(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.make_head_request") - def test_urls_cli_head(self, mock_make_head_request): + def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ @@ -753,6 +672,11 @@ def test_urls_cli_head(self, mock_make_head_request): {"get_request": 200}, {"head_request": 200}, ] + + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head_mock.json" ) @@ -781,7 +705,6 @@ def test_urls_cli_head(self, mock_make_head_request): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -842,10 +765,15 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.check_urls_purl") - def test_urls_details(self, mock_check_urls_purl): + def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): mock_check_urls_purl.return_value = "valid_but_not_fully_supported" + mock_read_log_file.return_value = [ + "WARNING = 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + ] + expected_data = { "headers": [ { @@ -857,7 +785,6 @@ def test_urls_details(self, mock_check_urls_purl): "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command" @@ -884,9 +811,6 @@ def test_urls_details(self, mock_check_urls_purl): "repo_url": { "url": "https://pypi.org/project/fetchcode/", }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, }, ], } @@ -899,7 +823,6 @@ def test_urls_details(self, mock_check_urls_purl): file="", command_name="urls", head=False, - unique=False, ) cli_test_utils.streamline_headers(expected_data["headers"]) cli_test_utils.streamline_headers(purl_urls["headers"]) @@ -961,59 +884,13 @@ def mock_requests_get_return_func(): results = purlcli.validate_purl(input_purl) assert mock_request_response == results - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_jsondecodeerror(self, mock_requests_get): - def json_decode_failure_exception(**kwargs): - raise json.decoder.JSONDecodeError("test", "[{}]", 0) - - mock_requests_get.return_value.json = json_decode_failure_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", json_decode_failure_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert ( - "json.decoder.JSONDecodeError for 'pkg:pypi/fetchcode': test: line 1 column 1 (char 0)" - in results - ) - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_exception(self, mock_requests_get): - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'pkg:pypi/fetchcode': \n" in results - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_none(self, mock_requests_get): - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = None - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'None': \n" in results - class TestPURLCLI_versions(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.collect_versions") @mock.patch("purldb_toolkit.purlcli.check_versions_purl") def test_versions_details_multiple( - self, - mock_check_versions_purl, - mock_collect_versions, + self, mock_check_versions_purl, mock_collect_versions, mock_read_log_file ): mock_check_versions_purl.side_effect = [ @@ -1030,40 +907,51 @@ def test_versions_details_multiple( { "purl": "pkg:pypi/fetchcode@0.1.0", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { "purl": "pkg:pypi/fetchcode@0.2.0", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { "purl": "pkg:pypi/fetchcode@0.3.0", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ], [ { "purl": "pkg:gem/bundler-sass@0.1.2", "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", + "release_date": "2013-12-11", } ], [ { "purl": "pkg:cargo/socksprox@0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", + "release_date": "2024-02-07", "version": "0.1.1", }, { "purl": "pkg:cargo/socksprox@0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", + "release_date": "2024-02-07", "version": "0.1.0", }, ], ] + mock_read_log_file.side_effect = [ + [], + [], + [ + "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", + ], + ["WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n"], + [], + ["WARNING - 'pkg:pypi/?fetchcode' not valid\n"], + ] + input_purls_and_expected_purl_data = [ [ ["pkg:pypi/fetchcode"], @@ -1078,32 +966,26 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], }, ], @@ -1120,21 +1002,15 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:gem/bundler-sass"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], + "purl": "pkg:gem/bundler-sass@0.1.2", + "version": "0.1.2", + "release_date": "2013-12-11", } ], }, @@ -1152,7 +1028,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:rubygems/bundler-sass"], "errors": [], "warnings": [ "'pkg:rubygems/bundler-sass' not supported with `versions` command" @@ -1175,7 +1050,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:nginx/nginx"], "errors": [], "warnings": [ "'pkg:nginx/nginx' not supported with `versions` command" @@ -1198,27 +1072,21 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:cargo/socksprox"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:cargo/socksprox", - "versions": [ - { - "purl": "pkg:cargo/socksprox@0.1.1", - "version": "0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", - }, - { - "purl": "pkg:cargo/socksprox@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", - }, - ], - } + "purl": "pkg:cargo/socksprox@0.1.1", + "version": "0.1.1", + "release_date": "2024-02-07", + }, + { + "purl": "pkg:cargo/socksprox@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07", + }, ], }, ], @@ -1235,7 +1103,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/?fetchcode"], "errors": [], "warnings": ["'pkg:pypi/?fetchcode' not valid"], } @@ -1248,11 +1115,13 @@ def test_versions_details_multiple( output = "" file = "" command_name = "versions" - unique = False for input_purl, expected_data in input_purls_and_expected_purl_data: purl_versions_data = purlcli.get_versions_details( - input_purl, output, file, unique, command_name + input_purl, + output, + file, + command_name, ) assert purl_versions_data == expected_data @@ -1269,17 +1138,17 @@ def test_versions_details( { "purl": "pkg:pypi/fetchcode@0.1.0", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { "purl": "pkg:pypi/fetchcode@0.2.0", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { "purl": "pkg:pypi/fetchcode@0.3.0", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ] @@ -1296,32 +1165,26 @@ def test_versions_details( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], } @@ -1330,13 +1193,11 @@ def test_versions_details( output = "" file = "" command_name = "versions" - unique = False purl_versions_data = purlcli.get_versions_details( input_purls, output, file, - unique, command_name, ) assert purl_versions_data == expected_data diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 929bc1af..a783b89e 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -10,11 +10,13 @@ import json import os from collections import OrderedDict +from unittest import mock import pytest from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -23,7 +25,8 @@ class TestPURLCLI_metadata(object): - def test_metadata_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_metadata_cli(self, mock_read_log_file): """ Test the `metadata` command with actual and expected JSON output files. @@ -31,6 +34,14 @@ def test_metadata_cli(self): because the `--output` values (paths) differ due to the use of temporary files, and therefore we test a list of relevant key-value pairs. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `metadata` command\n", + "WARNING - 'pkg:gem/rails' not supported with `metadata` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_metadata_output.json" ) @@ -76,7 +87,6 @@ def test_metadata_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -102,94 +112,12 @@ def test_metadata_cli(self): for output, expected in result_objects: assert output == expected - # NOTE: To avoid errors from the addition of new versions, we exclude - # "packages" from the result_objects list above and handle here. - # All other live-fetch tests are handled in a similar manner. - - compare_packages(expected_data, output_data, "metadata") - - def test_metadata_cli_unique(self): """ - Test the `metadata` command with actual and expected JSON output files - with the `--unique` flag included in the command. + NOTE: To avoid errors from the addition of new versions, we exclude + "packages" from the result_objects list above and handle here. All + other live-fetch tests are handled in a similar manner. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_metadata_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_metadata_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", - "--output", - actual_result_file, - "--unique", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - streamline_metadata_packages(output_data["packages"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - streamline_metadata_packages(expected_data["packages"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), - ] - - for output, expected in result_objects: - assert output == expected - - compare_packages(expected_data, output_data, "metadata") + compare_packages(expected_data, output_data) def test_metadata_cli_duplicate_input_sources(self): """ @@ -221,430 +149,190 @@ def test_metadata_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ("download_url", None), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - ], - } - ], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "metadata", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "options": { - "command": "metadata", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'check_existence' is not supported for " - "'pkg:rubygems/bundler-sass'", - ], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:rubygems/bundler-sass"), - ("type", "rubygems"), - ("namespace", None), - ("name", "bundler-sass"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "http://github.com/vogelbek/bundler-sass", - ), - ( - "download_url", - "https://rubygems.org/gems/bundler-sass-0.1.2.gem", - ), - ( - "api_url", - "https://rubygems.org/api/v1/gems/bundler-sass.json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", ["MIT"]), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ) - ], - } - ], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "metadata", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "metadata", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "metadata", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], + def test_metadata_details(self): + expected_data = { + "headers": [ { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "metadata", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } - ], - "packages": [], - }, - ), - ], - ) - def test_metadata_details(self, test_input, expected): - """ - Test the `metadata` nested function, `get_metadata_details()`. - """ - purl_metadata = purlcli.get_metadata_details( - test_input, - output="", - file="", - command_name="metadata", - unique=False, + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": ["pkg:pypi/fetchcode"], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [], + } + ], + "packages": [ + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", None), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + ], + } + + input_purls = ["pkg:pypi/fetchcode"] + output = "" + file = "" + command_name = "metadata" + + purl_metadata_data = purlcli.get_metadata_details( + input_purls, + output, + file, + command_name, ) - cli_test_utils.streamline_headers(purl_metadata["headers"]) - streamline_metadata_packages(purl_metadata["packages"]) - cli_test_utils.streamline_headers(expected["headers"]) - streamline_metadata_packages(expected["packages"]) + cli_test_utils.streamline_headers(purl_metadata_data["headers"]) + streamline_metadata_packages(purl_metadata_data["packages"]) - assert purl_metadata["headers"] == expected["headers"] + cli_test_utils.streamline_headers(expected_data["headers"]) + streamline_metadata_packages(expected_data["packages"]) - compare_packages(expected, purl_metadata, "metadata") + assert purl_metadata_data["headers"] == expected_data["headers"] + compare_packages(expected_data, purl_metadata_data) @pytest.mark.parametrize( "test_input,expected", @@ -684,113 +372,14 @@ def test_check_metadata_purl(self, test_input, expected): assert purl_metadata == expected @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", + "test_input,expected", [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), ( [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", + "pkg:gem/bundler-sass", + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", ], [ { @@ -806,12 +395,6 @@ def test_normalize_purls( ], "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.2.0", "warnings": [ @@ -822,79 +405,18 @@ def test_normalize_purls( ), ], ) - def test_construct_headers(self, test_input, expected): - metadata_headers = purlcli.construct_headers( - test_input, - output="", - file="", - command_name="metadata", - head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, - ) - cli_test_utils.streamline_headers(expected) - cli_test_utils.streamline_headers(metadata_headers) - - assert metadata_headers == expected + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_construct_headers(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", + ] - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "--unique": True, - "command": "metadata", - }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", - ], - } - ], - ), - ], - ) - def test_construct_headers_unique(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), - ], - unique=True, purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) cli_test_utils.streamline_headers(expected) @@ -904,10 +426,30 @@ def test_construct_headers_unique(self, test_input, expected): class TestPURLCLI_urls(object): - def test_urls_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_cli(self, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", + "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", + "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", + "WARNING - 'abcdefg' not valid\n", + "WARNING - 'pkg/abc' not valid\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output.json" ) @@ -973,7 +515,6 @@ def test_urls_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1000,107 +541,30 @@ def test_urls_cli(self): for output, expected in result_objects: assert output == expected - def test_urls_cli_unique(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_cli_head(self, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_urls_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_urls_output_unique.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:pypi/dejacode", - "--purl", - "pkg:pypi/dejacode@5.0.0", - "--purl", - "pkg:pypi/dejacode@5.0.0?os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:cargo/socksprox", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", - "pkg:gem/bundler-sass", - "--purl", - "pkg:rubygems/bundler-sass", - "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", - "pkg:nuget/auth0-aspnet@1.1.0", - "--output", - actual_result_file, - "--unique", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - (output_data["packages"], expected_data["packages"]), + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", + "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", + "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", + "WARNING - 'abcdefg' not valid\n", + "WARNING - 'pkg/abc' not valid\n", ] - for output, expected in result_objects: - assert output == expected - - def test_urls_cli_head(self): - """ - Test the `urls` command with actual and expected JSON output files. - """ expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head.json" ) @@ -1167,7 +631,6 @@ def test_urls_cli_head(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1228,275 +691,102 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_details(self, mock_read_log_file): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'\n", + ] + + input_purls = [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ] + output = "" + file = "" + command_name = "urls" + head = False + + purl_urls_data = purlcli.get_urls_details( + input_purls, + output, + file, + head, + command_name, + ) + + expected_data = { + "headers": [ { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, - }, + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [ + "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", + "'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'", ], - }, - ), - ( - ["pkg:pypi/fetchcode@10.0.0"], + } + ], + "packages": [ { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode@10.0.0"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode@10.0.0"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@10.0.0' does not exist in the upstream repo", - ], - } + "purl": "pkg:pypi/fetchcode@0.3.0", + "download_url": {"url": None}, + "inferred_urls": [ + {"url": "https://pypi.org/project/fetchcode/0.3.0/"} ], - "packages": [], + "repo_download_url": {"url": None}, + "repo_download_url_by_package_type": {"url": None}, + "repo_url": {"url": "https://pypi.org/project/fetchcode/0.3.0/"}, }, - ), - ( - ["pkg:gem/bundler-sass"], { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - }, + "purl": "pkg:gem/bundler@2.3.23", + "download_url": { + "url": "https://rubygems.org/downloads/bundler-2.3.23.gem" + }, + "inferred_urls": [ + {"url": "https://rubygems.org/gems/bundler/versions/2.3.23"}, + {"url": "https://rubygems.org/downloads/bundler-2.3.23.gem"}, ], + "repo_download_url": {"url": None}, + "repo_download_url_by_package_type": {"url": None}, + "repo_url": { + "url": "https://rubygems.org/gems/bundler/versions/2.3.23" + }, }, - ), - ( - ["pkg:rubygems/bundler-sass"], { - "headers": [ + "purl": "pkg:github/istio/istio@1.20.2", + "download_url": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "inferred_urls": [ + {"url": "https://github.com/istio/istio/tree/1.20.2"}, { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:rubygems/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" }, ], + "repo_download_url": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "repo_download_url_by_package_type": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "repo_url": {"url": "https://github.com/istio/istio/tree/1.20.2"}, }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "urls", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `urls` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "urls", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "urls", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "urls", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } - ], - "packages": [], - }, - ), - ], - ) - def test_urls_details(self, test_input, expected): - """ - Test the `urls` nested function, `get_urls_details()`. - """ - purl_urls = purlcli.get_urls_details( - test_input, - output="", - file="", - command_name="urls", - head=False, - unique=False, - ) - cli_test_utils.streamline_headers(expected["headers"]) - cli_test_utils.streamline_headers(purl_urls["headers"]) + ], + } - assert purl_urls == expected + assert purl_urls_data == expected_data @pytest.mark.parametrize( "test_input,expected", @@ -1511,141 +801,76 @@ def test_urls_details(self, test_input, expected): ), ( ["pkg:rubygems/bundler-sass"], - None, - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_urls_purl(self, test_input, expected): - purl_urls = purlcli.check_urls_purl(test_input[0]) - assert purl_urls == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["https://pypi.org/project/fetchcode/"], - {"get_request": 200, "head_request": 200}, - ), - ( - [None], - {"get_request": "N/A", "head_request": "N/A"}, - ), - ( - ["https://crates.io/crates/banquo"], - {"get_request": 404, "head_request": 404}, - ), - ( - ["https://crates.io/crates/socksprox"], - {"get_request": 404, "head_request": 404}, + "check_existence_not_supported", ), ( - ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], - {"get_request": 200, "head_request": 404}, - ), - ], - ) - def test_make_head_request(self, test_input, expected): - purl_status_code = purlcli.make_head_request(test_input[0]) - - assert purl_status_code == expected - - -class TestPURLCLI_validate(object): - def test_validate_cli(self): - """ - Test the `validate` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_validate_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.validate, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - - result_objects = [ + ["pkg:nginx/nginx"], + "valid_but_not_supported", + ), ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], + ["pkg:pypi/zzzzz"], + "not_in_upstream_repo", ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], + ["pkg:pypi/?fetchcode"], + "not_valid", ), ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], + ["zzzzz"], + "not_valid", ), + ], + ) + def test_check_urls_purl(self, test_input, expected): + purl_urls = purlcli.check_urls_purl(test_input[0]) + assert purl_urls == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], + ["https://pypi.org/project/fetchcode/"], + {"get_request": 200, "head_request": 200}, ), ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], + [None], + {"get_request": "N/A", "head_request": "N/A"}, ), ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], + ["https://crates.io/crates/banquo"], + {"get_request": 404, "head_request": 404}, ), - (output_data["packages"], expected_data["packages"]), - ] + ( + ["https://crates.io/crates/socksprox"], + {"get_request": 404, "head_request": 404}, + ), + ( + ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], + {"get_request": 200, "head_request": 404}, + ), + ], + ) + def test_make_head_request(self, test_input, expected): + purl_status_code = purlcli.make_head_request(test_input[0]) + + assert purl_status_code == expected - for output, expected in result_objects: - assert output == expected - def test_validate_cli_unique(self): +class TestPURLCLI_validate(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_validate_cli(self, mock_read_log_file): """ - Test the `validate` command with actual and expected JSON output files - with the `--unique` flag included in the command. + Test the `validate` command with actual and expected JSON output files. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'check_existence' is not supported for 'pkg:nginx/nginx'\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", + ] + expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output_unique.json" + "purlcli/expected_validate_output.json" ) actual_result_file = test_env.get_temp_file("actual_validate_output.json") options = [ @@ -1669,7 +894,6 @@ def test_validate_cli_unique(self): "pkg:rubygems/rails", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.validate, options, catch_exceptions=False) @@ -1686,7 +910,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1707,10 +930,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), (output_data["packages"], expected_data["packages"]), ] @@ -1869,86 +1088,19 @@ def test_validate_purl_strip(self, test_input, expected): class TestPURLCLI_versions(object): - def test_versions_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_versions_cli(self, mock_read_log_file): """ Test the `versions` command with actual and expected JSON output files. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_versions_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_versions_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:hex/coherence@0.1.0", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", ] - for output, expected in result_objects: - assert output == expected - - compare_packages(expected_data, output_data, "versions") - - def test_versions_cli_unique(self): - """ - Test the `versions` command with actual and expected JSON output files - with the `--unique` flag included in the command. - """ expected_result_file = test_env.get_test_loc( - "purlcli/expected_versions_output_unique.json" + "purlcli/expected_versions_output.json" ) actual_result_file = test_env.get_temp_file("actual_versions_output.json") options = [ @@ -1970,7 +1122,6 @@ def test_versions_cli_unique(self): "pkg:hex/coherence@0.1.0", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) @@ -1989,7 +1140,6 @@ def test_versions_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -2010,22 +1160,26 @@ def test_versions_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), ] for output, expected in result_objects: assert output == expected - compare_packages(expected_data, output_data, "versions") + compare_packages(expected_data, output_data) @pytest.mark.parametrize( "test_input,expected", [ ( - ["pkg:pypi/fetchcode"], + [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:rubygems/bundler-sass", + "pkg:nginx/nginx", + "pkg:pypi/zzzzz", + "pkg:pypi/?fetchcode", + "zzzzz", + ], { "headers": [ { @@ -2033,197 +1187,78 @@ def test_versions_cli_unique(self): "tool_version": "0.2.0", "options": { "command": "versions", - "--purl": ["pkg:pypi/fetchcode"], + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:rubygems/bundler-sass", + "pkg:nginx/nginx", + "pkg:pypi/zzzzz", + "pkg:pypi/?fetchcode", + "zzzzz", + ], "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "release_date": "2024-03-12 07:01:29.239299+00:00", - "version": "0.4.0", - }, + "warnings": [ + "'pkg:rubygems/bundler-sass' not supported with `versions` command", + "'pkg:nginx/nginx' not supported with `versions` command", + "'pkg:pypi/zzzzz' does not exist in the upstream repo", + "'pkg:pypi/?fetchcode' not valid", + "'zzzzz' not valid", ], } ], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:gem/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:gem/bundler-sass"], - "errors": [], - "warnings": [], - } - ], "packages": [ { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], - } - ], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:nginx/nginx"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:nginx/nginx"], - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/zzzzz"], - "errors": [], - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/?fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/?fetchcode"], - "errors": [], - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.4.0", + "release_date": "2024-03-12", + "version": "0.4.0", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["zzzzz"], - "errors": [], - "warnings": ["'zzzzz' not valid"], - } + "purl": "pkg:gem/bundler-sass@0.1.2", + "version": "0.1.2", + "release_date": "2013-12-11", + }, ], - "packages": [], }, ), ], ) - def test_versions_details(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_versions_details(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", + "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", + "WARNING - 'pkg:pypi/zzzzz' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/?fetchcode' not valid\n", + "WARNING - 'zzzzz' not valid\n", + ] + output = "" file = "" command_name = "versions" - unique = False purl_versions = purlcli.get_versions_details( - test_input, output, file, unique, command_name + test_input, + output, + file, + command_name, ) cli_test_utils.streamline_headers(purl_versions["headers"]) @@ -2231,7 +1266,7 @@ def test_versions_details(self, test_input, expected): assert purl_versions["headers"] == expected["headers"] - compare_packages(expected, purl_versions, "versions") + compare_packages(expected, purl_versions) @pytest.mark.parametrize( "test_input,expected", @@ -2284,7 +1319,7 @@ def streamline_metadata_packages(packages): hle.pop("download_url", None) -def compare_packages(expected_data, actual_data, nested_key): +def compare_packages(expected_data, actual_data): """ Compare the expected and actual data nested inside the `packages` field returned from a live-fetch query and assert that expected data from prior @@ -2301,7 +1336,4 @@ def compare_packages(expected_data, actual_data, nested_key): actual.append(actual_pkg) for expected_entry in expected: - for actual_entry in actual: - if expected_entry["purl"] == actual_entry["purl"]: - for expected_sibling in expected_entry[nested_key]: - assert expected_sibling in actual_entry[nested_key] + assert expected_entry in actual From f49eff87ae9a4226584096498836de7d5015f6a2 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 28 May 2024 13:32:27 -0700 Subject: [PATCH 2/9] Add purlcli RTD details, incorporate latest feedback, fix failing tests #365 #249 Reference: https://github.com/nexB/purldb/issues/365 Reference: https://github.com/nexB/purldb/issues/249 Signed-off-by: John M. Horan --- purldb-toolkit/README.rst | 513 +++++++++++++++++- purldb-toolkit/src/purldb_toolkit/purlcli.py | 108 +++- .../data/purlcli/expected_urls_output.json | 232 ++------ .../purlcli/expected_urls_output_head.json | 108 +--- .../expected_urls_output_head_mock.json | 9 +- purldb-toolkit/tests/test_purlcli.py | 21 +- purldb-toolkit/tests/test_purlcli_live.py | 67 +-- 7 files changed, 685 insertions(+), 373 deletions(-) diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index 4ac94a98..3662cae3 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -1,12 +1,17 @@ purldb-toolkit ============== +.. contents:: :local: + :depth: 7 + + + purldb-toolkit is command line utility and library to use the PurlDB, its API and various related libraries. The ``purlcli`` command acts as a client to the PurlDB REST API end point(s) to expose PURL services. It serves both as a tool, as a library and as an example on how to use the services programmatically. - + Installation ------------ @@ -20,84 +25,541 @@ Use this command to get basic help:: $ purlcli --help Usage: purlcli [OPTIONS] COMMAND [ARGS]... - + Return information from a PURL. - + Options: --help Show this message and exit. - + Commands: metadata Given one or more PURLs, for each PURL, return a mapping of... urls Given one or more PURLs, for each PURL, return a list of all... - validate Check the syntax of one or more PURLs. + validate Check the syntax and upstream repo status of one or more PURLs. versions Given one or more PURLs, return a list of all known versions... And the following subcommands: -- Validate a PURL:: +'validate' -- validate a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: none $ purlcli validate --help Usage: purlcli validate [OPTIONS] - - Check the syntax of one or more PURLs. - + + Check the syntax and upstream repo status of one or more PURLs. + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write validation output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli validate --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "validate", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:nginx/nginx@0.8.9'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo." + }, + { + "purl": "pkg:nginx/nginx@0.8.9", + "valid": true, + "exists": null, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type." + } + ] + } + + +**Submit multiple PURLs using a .txt file:** + +.. code-block:: none + + purlcli validate --file --output + +*Sample input.txt:* + +.. code-block:: console + + pkg:npm/canonical-path@1.0.0 + pkg:nginx/nginx@0.8.9 + + +Notes +####### + +``validate`` calls the ``public.purldb.io/api/validate/`` endpoint. + + +---- + + +'versions' -- collect package versions for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: none -- Collect package versions for a PURL:: - $ purlcli versions --help Usage: purlcli versions [OPTIONS] - + Given one or more PURLs, return a list of all known versions for each PURL. - - Version information is not needed in submitted PURLs and if included will be - removed before processing. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write versions output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli versions --purl pkg:npm/canonical-path --purl pkg:nginx/nginx --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": [ + "pkg:npm/canonical-path", + "pkg:nginx/nginx" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx' not supported with `versions` command" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@0.0.1", + "version": "0.0.1", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@0.0.2", + "version": "0.0.2", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@1.0.0", + "version": "1.0.0", + "release_date": "2018-10-24" + } + ] + } + -- Collect package metadata for a PURL:: +Notes +####### + +``versions`` calls ``versions()`` from `fetchcode/package_versions.py `__. + +Version information is not needed in submitted PURLs and, if included, will be removed before processing. + + +---- + + +'metadata' -- collect package metadata for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console $ purlcli metadata --help Usage: purlcli metadata [OPTIONS] - + Given one or more PURLs, for each PURL, return a mapping of metadata fetched from the fetchcode package.py info() function. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write meta output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. --help Show this message and exit. - -- Collect package URLs for a PURL:: +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli metadata --purl pkg:openssl/openssl@3.0.6 --purl pkg:nginx/nginx@0.8.9 --purl pkg:gnu/glibc@2.38 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": [ + "pkg:openssl/openssl@3.0.6", + "pkg:nginx/nginx@0.8.9", + "pkg:gnu/glibc@2.38" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:openssl/openssl@3.0.6'", + "'pkg:nginx/nginx@0.8.9' not supported with `metadata` command", + "'check_existence' is not supported for 'pkg:gnu/glibc@2.38'" + ] + } + ], + "packages": [ + { + "purl": "pkg:openssl/openssl@3.0.6", + "type": "openssl", + "namespace": null, + "name": "openssl", + "version": "3.0.6", + "qualifiers": {}, + "subpath": null, + "primary_language": "C", + "description": null, + "release_date": "2022-10-11T12:39:09", + "parties": [], + "keywords": [], + "homepage_url": "https://www.openssl.org", + "download_url": "https://github.com/openssl/openssl/archive/refs/tags/openssl-3.0.6.tar.gz", + "api_url": "https://api.github.com/repos/openssl/openssl", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/openssl/openssl/issues", + "code_view_url": "https://github.com/openssl/openssl", + "vcs_url": "git://github.com/openssl/openssl.git", + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:gnu/glibc@2.38", + "type": "gnu", + "namespace": null, + "name": "glibc", + "version": "2.38", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": "2023-07-31T17:34:00", + "parties": [], + "keywords": [], + "homepage_url": "https://ftp.gnu.org/pub/gnu/glibc/", + "download_url": "https://ftp.gnu.org/pub/gnu/glibc/glibc-2.38.tar.gz", + "api_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ] + } + + +Notes +####### + +``metadata`` calls ``info()`` from `fetchcode/package.py `__. + +The intended output for each PURL type supported by the ``metadata`` command is + +- an input PURL with a version: output the metadata for the input version +- an input PURL with no version: output a list of the metadata for all versions + +The output of the various PURL types currently supported in `fetchcode/package.py `__ varies from type to type at the moment -- the underlying functions will be updated as needed so that all produce the intended output for input PURLs with and without a version. + + +---- + + +'urls' -- collect package URLs for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console $ purlcli urls --help Usage: purlcli urls [OPTIONS] - + Given one or more PURLs, for each PURL, return a list of all known URLs fetched from the packageurl-python purl2url.py code. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write urls output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. --head Validate each URL's existence with a head request. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "inferred_urls": [ + "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz" + ], + "repository_download_url": null, + "repository_homepage_url": "https://www.npmjs.com/package/canonical-path/v/1.0.0" + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "inferred_urls": [ + "https://rubygems.org/gems/rails/versions/7.0.0", + "https://rubygems.org/downloads/rails-7.0.0.gem" + ], + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/rails/versions/7.0.0" + } + ] + } + + +**Include head and get requests:** + +``--head`` + +.. code-block:: none + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output --head + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--head": true, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + }, + "inferred_urls": [ + { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + "inferred_urls": [ + { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + } + ] + } + + +Notes +####### + +- None atm. + + +Testing +------- + +Run all purldb tests: + +.. code-block:: none + + make test + +Run all purlcli non-live tests (i.e., no live network calls): + +.. code-block:: none + + DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli.py + +Run all purlcli live tests (i.e., check actual API endpoints etc.) + +.. code-block:: none + + DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli_live.py --run_live_fetch + Funding ------- @@ -132,4 +594,3 @@ See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license tex See https://github.com/nexB/purldb for support or download. See https://aboutcode.org for more information about nexB OSS projects. - diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 917c1008..5997e2a5 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -101,7 +101,7 @@ def get_metadata_details(purls, output, file, command_name): metadata_collection = collect_metadata(purl) metadata_details["packages"].extend(metadata_collection) - print(f"\nmetadata_warnings = {metadata_warnings}") + # print(f"\nmetadata_warnings = {metadata_warnings}") metadata_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, @@ -349,26 +349,64 @@ def get_urls_details(purls, output, file, head, command_name): url_purl = PackageURL.from_string(purl) - url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} + url_detail["download_url"] = purl2url.get_download_url(purl) + if head: + url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} + url_detail["inferred_urls"] = [ - {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + inferred for inferred in purl2url.get_inferred_urls(purl) ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} - url_detail["repo_download_url_by_package_type"] = {"url": None} - if url_purl.version: - url_detail["repo_download_url_by_package_type"] = { - "url": purl2url.get_repo_download_url_by_package_type( - url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - ) + if head: + url_detail["inferred_urls"] = [ + {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + ] + + # url_detail["repo_download_url"] = purl2url.get_repo_download_url(purl) + url_detail["repository_download_url"] = purl2url.get_repo_download_url(purl) + if head: + # url_detail["repo_download_url"] = { + url_detail["repository_download_url"] = { + "url": purl2url.get_repo_download_url(purl) } - url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} + + # TODO delete this -- seems always same as the renamed repository_download_url + # package_type_url = None + # # url_detail["repo_download_url_by_package_type"] = {"url": None} + # url_detail["repo_download_url_by_package_type"] = package_type_url + # if head: + # url_detail["repo_download_url_by_package_type"] = {"url": package_type_url} + + # if url_purl.version: + # package_type_url = purl2url.get_repo_download_url_by_package_type( + # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + # ) + # url_detail["repo_download_url_by_package_type"] = package_type_url + # # url_detail["repo_download_url_by_package_type"] = { + # # "url": purl2url.get_repo_download_url_by_package_type( + # # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + # # ) + # # } + # if head: + # url_detail["repo_download_url_by_package_type"] = { + # "url": package_type_url + # } + + # url_detail["repo_url"] = purl2url.get_repo_url(purl) + # if head: + # url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} + url_detail["repository_homepage_url"] = purl2url.get_repo_url(purl) + if head: + url_detail["repository_homepage_url"] = {"url": purl2url.get_repo_url(purl)} url_list = [ "download_url", # "inferred_urls" has to be handled separately because it has a nested list - "repo_download_url", - "repo_download_url_by_package_type", - "repo_url", + # "repo_download_url", + "repository_download_url", + # TODO delete this -- seems always same as the renamed repository_download_url + # "repo_download_url_by_package_type", + # "repo_url", + "repository_homepage_url", ] if head: for purlcli_url in url_list: @@ -543,7 +581,33 @@ def get_validate_details(purls, output, file, command_name): ]: validate_warnings[purl] = validated_purl_status if validated_purl_status: - validate_details["packages"].append(validate_purl(purl)) + # TODO: move the `purl` key to the top. xxx + original_validate_purl = validate_purl(purl) + print(f"\noriginal_validate_purl = {original_validate_purl}") + + # === + # # print(f"\nresponse = {response}") + # # Create a new dict with `purl` at the top. + # # response = {"purl": response.pop("purl"), **response} + + reordered_validate_purl = { + "purl": original_validate_purl.pop("purl"), + **original_validate_purl, + } + print(f"\nreordered_validate_purl = {reordered_validate_purl}") + + # # print(f"\nupdated response = {response}") + # # or + # ordered_response = OrderedDict(response) + # purl_value = ordered_response.pop("purl") + # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) + # response = dict(ordered_response) + # # print(f"updated response = {response}") + # === + + # validate_details["packages"].append(validate_purl(purl)) + # validate_details["packages"].append(validate_purl(purl)) + validate_details["packages"].append(reordered_validate_purl) validate_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, @@ -607,6 +671,20 @@ def validate_purl(purl): try: response = requests.get(api_query, params=request_body).json() + # ZAP 2024-05-27 Monday 19:41:16. Getting a test error: AssertionError: assert None == {'errors': {'purl': ['This field is required.']}} + # ZAP so maybe do the move-purl-key process after the return below? + # # TODO: move the `purl` key to the top of the dict. + # # print(f"\nresponse = {response}") + # # Create a new dict with `purl` at the top. + # # response = {"purl": response.pop("purl"), **response} + # # print(f"\nupdated response = {response}") + # # or + # ordered_response = OrderedDict(response) + # purl_value = ordered_response.pop("purl") + # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) + # response = dict(ordered_response) + # # print(f"updated response = {response}") + except json.decoder.JSONDecodeError as e: logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") except Exception as e: diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index dac9392b..c2bf50d7 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -54,246 +54,112 @@ "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } + "https://pypi.org/project/fetchcode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/" }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "https://pypi.org/project/fetchcode/0.3.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/" }, { "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } + "https://pypi.org/project/dejacode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/" }, { "purl": "pkg:pypi/dejacode@5.0.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0?os=windows", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } + "https://crates.io/crates/banquo" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/banquo" }, { "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } + "https://crates.io/crates/socksprox" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/socksprox" }, { "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:nuget/auth0-aspnet@1.1.0", - "download_url": { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - }, + "download_url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0", "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - } + "https://www.nuget.org/packages/auth0-aspnet/1.1.0", + "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - } + "repository_download_url": null, + "repository_homepage_url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index b256c973..2e8de8c1 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -67,17 +67,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -97,17 +92,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/0.3.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -127,17 +117,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -157,17 +142,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -187,17 +167,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -217,17 +192,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -247,17 +217,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -277,17 +242,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://crates.io/crates/banquo", "get_request_status_code": 404, "head_request_status_code": 404 @@ -307,17 +267,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://crates.io/crates/socksprox", "get_request_status_code": 404, "head_request_status_code": 404 @@ -337,17 +292,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -367,17 +317,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -402,17 +347,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", "get_request_status_code": 200, "head_request_status_code": 404 diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 16578130..7de378fd 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -33,17 +33,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 7a8c90de..43def355 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -663,8 +663,6 @@ def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): {"head_request": "N/A"}, {"get_request": "N/A"}, {"head_request": "N/A"}, - {"get_request": "N/A"}, - {"head_request": "N/A"}, {"get_request": 200}, {"head_request": 200}, {"get_request": 200}, @@ -794,23 +792,12 @@ def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, + "download_url": None, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } + "https://pypi.org/project/fetchcode/", ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/", }, ], } diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index a783b89e..372b7bab 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -175,9 +175,6 @@ def test_metadata_details(self): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -200,6 +197,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -211,9 +211,6 @@ def test_metadata_details(self): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -236,6 +233,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -247,9 +247,6 @@ def test_metadata_details(self): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -272,6 +269,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -283,9 +283,6 @@ def test_metadata_details(self): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -308,6 +305,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), ], @@ -741,47 +741,32 @@ def test_urls_details(self, mock_read_log_file): "packages": [ { "purl": "pkg:pypi/fetchcode@0.3.0", - "download_url": {"url": None}, + "download_url": None, "inferred_urls": [ - {"url": "https://pypi.org/project/fetchcode/0.3.0/"} + "https://pypi.org/project/fetchcode/0.3.0/", ], - "repo_download_url": {"url": None}, - "repo_download_url_by_package_type": {"url": None}, - "repo_url": {"url": "https://pypi.org/project/fetchcode/0.3.0/"}, + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/", }, { "purl": "pkg:gem/bundler@2.3.23", - "download_url": { - "url": "https://rubygems.org/downloads/bundler-2.3.23.gem" - }, + "download_url": "https://rubygems.org/downloads/bundler-2.3.23.gem", "inferred_urls": [ - {"url": "https://rubygems.org/gems/bundler/versions/2.3.23"}, - {"url": "https://rubygems.org/downloads/bundler-2.3.23.gem"}, + "https://rubygems.org/gems/bundler/versions/2.3.23", + "https://rubygems.org/downloads/bundler-2.3.23.gem", ], - "repo_download_url": {"url": None}, - "repo_download_url_by_package_type": {"url": None}, - "repo_url": { - "url": "https://rubygems.org/gems/bundler/versions/2.3.23" - }, + "repository_download_url": None, + "repository_homepage_url": "https://rubygems.org/gems/bundler/versions/2.3.23", }, { "purl": "pkg:github/istio/istio@1.20.2", - "download_url": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, + "download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", "inferred_urls": [ - {"url": "https://github.com/istio/istio/tree/1.20.2"}, - { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, + "https://github.com/istio/istio/tree/1.20.2", + "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", ], - "repo_download_url": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, - "repo_download_url_by_package_type": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, - "repo_url": {"url": "https://github.com/istio/istio/tree/1.20.2"}, + "repository_download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", + "repository_homepage_url": "https://github.com/istio/istio/tree/1.20.2", }, ], } From f3846dbb8ea9741995751907f9a75af09bc33428 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 28 May 2024 19:32:17 -0700 Subject: [PATCH 3/9] Remove unneeded comments #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 70 +------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 5997e2a5..28d99e7d 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -101,8 +101,6 @@ def get_metadata_details(purls, output, file, command_name): metadata_collection = collect_metadata(purl) metadata_details["packages"].extend(metadata_collection) - # print(f"\nmetadata_warnings = {metadata_warnings}") - metadata_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, duplicate_purls=duplicate_purls, @@ -361,51 +359,19 @@ def get_urls_details(purls, output, file, head, command_name): {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) ] - # url_detail["repo_download_url"] = purl2url.get_repo_download_url(purl) url_detail["repository_download_url"] = purl2url.get_repo_download_url(purl) if head: - # url_detail["repo_download_url"] = { url_detail["repository_download_url"] = { "url": purl2url.get_repo_download_url(purl) } - # TODO delete this -- seems always same as the renamed repository_download_url - # package_type_url = None - # # url_detail["repo_download_url_by_package_type"] = {"url": None} - # url_detail["repo_download_url_by_package_type"] = package_type_url - # if head: - # url_detail["repo_download_url_by_package_type"] = {"url": package_type_url} - - # if url_purl.version: - # package_type_url = purl2url.get_repo_download_url_by_package_type( - # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - # ) - # url_detail["repo_download_url_by_package_type"] = package_type_url - # # url_detail["repo_download_url_by_package_type"] = { - # # "url": purl2url.get_repo_download_url_by_package_type( - # # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - # # ) - # # } - # if head: - # url_detail["repo_download_url_by_package_type"] = { - # "url": package_type_url - # } - - # url_detail["repo_url"] = purl2url.get_repo_url(purl) - # if head: - # url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} url_detail["repository_homepage_url"] = purl2url.get_repo_url(purl) if head: url_detail["repository_homepage_url"] = {"url": purl2url.get_repo_url(purl)} url_list = [ "download_url", - # "inferred_urls" has to be handled separately because it has a nested list - # "repo_download_url", "repository_download_url", - # TODO delete this -- seems always same as the renamed repository_download_url - # "repo_download_url_by_package_type", - # "repo_url", "repository_homepage_url", ] if head: @@ -581,32 +547,12 @@ def get_validate_details(purls, output, file, command_name): ]: validate_warnings[purl] = validated_purl_status if validated_purl_status: - # TODO: move the `purl` key to the top. xxx + # Move the `purl` key to the top. original_validate_purl = validate_purl(purl) - print(f"\noriginal_validate_purl = {original_validate_purl}") - - # === - # # print(f"\nresponse = {response}") - # # Create a new dict with `purl` at the top. - # # response = {"purl": response.pop("purl"), **response} - reordered_validate_purl = { "purl": original_validate_purl.pop("purl"), **original_validate_purl, } - print(f"\nreordered_validate_purl = {reordered_validate_purl}") - - # # print(f"\nupdated response = {response}") - # # or - # ordered_response = OrderedDict(response) - # purl_value = ordered_response.pop("purl") - # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) - # response = dict(ordered_response) - # # print(f"updated response = {response}") - # === - - # validate_details["packages"].append(validate_purl(purl)) - # validate_details["packages"].append(validate_purl(purl)) validate_details["packages"].append(reordered_validate_purl) validate_details["headers"] = construct_headers( @@ -671,20 +617,6 @@ def validate_purl(purl): try: response = requests.get(api_query, params=request_body).json() - # ZAP 2024-05-27 Monday 19:41:16. Getting a test error: AssertionError: assert None == {'errors': {'purl': ['This field is required.']}} - # ZAP so maybe do the move-purl-key process after the return below? - # # TODO: move the `purl` key to the top of the dict. - # # print(f"\nresponse = {response}") - # # Create a new dict with `purl` at the top. - # # response = {"purl": response.pop("purl"), **response} - # # print(f"\nupdated response = {response}") - # # or - # ordered_response = OrderedDict(response) - # purl_value = ordered_response.pop("purl") - # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) - # response = dict(ordered_response) - # # print(f"updated response = {response}") - except json.decoder.JSONDecodeError as e: logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") except Exception as e: From c8fef492ec4190004cfa2e67a360d72242165c4e Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 30 May 2024 11:27:52 -0700 Subject: [PATCH 4/9] Revert purldb-toolkit to pre-RTD update #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/README.rst | 486 +------------------------------------- 1 file changed, 12 insertions(+), 474 deletions(-) diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index 3662cae3..cb70a999 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -1,11 +1,6 @@ purldb-toolkit ============== -.. contents:: :local: - :depth: 7 - - - purldb-toolkit is command line utility and library to use the PurlDB, its API and various related libraries. The ``purlcli`` command acts as a client to the PurlDB REST API end point(s) to expose PURL services. @@ -34,21 +29,18 @@ Use this command to get basic help:: Commands: metadata Given one or more PURLs, for each PURL, return a mapping of... urls Given one or more PURLs, for each PURL, return a list of all... - validate Check the syntax and upstream repo status of one or more PURLs. + validate Check the syntax of one or more PURLs. versions Given one or more PURLs, return a list of all known versions... And the following subcommands: -'validate' -- validate a PURL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: none +- Validate a PURL:: $ purlcli validate --help Usage: purlcli validate [OPTIONS] - Check the syntax and upstream repo status of one or more PURLs. + Check the syntax of one or more PURLs. Options: --purl TEXT PackageURL or PURL. @@ -56,163 +48,25 @@ And the following subcommands: --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. -Examples -######## - -**Submit multiple PURLs using the command line:** - -.. code-block:: none - - purlcli validate --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --output - -*Sample output:* - -.. code-block:: console - - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "validate", - "--purl": [ - "pkg:npm/canonical-path@1.0.0", - "pkg:nginx/nginx@0.8.9" - ], - "--file": null, - "--output": "" - }, - "errors": [], - "warnings": [ - "'check_existence' is not supported for 'pkg:nginx/nginx@0.8.9'" - ] - } - ], - "packages": [ - { - "purl": "pkg:npm/canonical-path@1.0.0", - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo." - }, - { - "purl": "pkg:nginx/nginx@0.8.9", - "valid": true, - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type." - } - ] - } - - -**Submit multiple PURLs using a .txt file:** - -.. code-block:: none - purlcli validate --file --output - -*Sample input.txt:* - -.. code-block:: console - - pkg:npm/canonical-path@1.0.0 - pkg:nginx/nginx@0.8.9 - - -Notes -####### - -``validate`` calls the ``public.purldb.io/api/validate/`` endpoint. - - ----- - - -'versions' -- collect package versions for a PURL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: none +- Collect package versions for a PURL:: $ purlcli versions --help Usage: purlcli versions [OPTIONS] Given one or more PURLs, return a list of all known versions for each PURL. + Version information is not needed in submitted PURLs and if included will be + removed before processing. + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write versions output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. -Examples -######## - -**Submit multiple PURLs using the command line:** -.. code-block:: none - - purlcli versions --purl pkg:npm/canonical-path --purl pkg:nginx/nginx --output - -*Sample output:* - -.. code-block:: console - - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": [ - "pkg:npm/canonical-path", - "pkg:nginx/nginx" - ], - "--file": null, - "--output": "" - }, - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:npm/canonical-path@0.0.1", - "version": "0.0.1", - "release_date": "2013-12-19" - }, - { - "purl": "pkg:npm/canonical-path@0.0.2", - "version": "0.0.2", - "release_date": "2013-12-19" - }, - { - "purl": "pkg:npm/canonical-path@1.0.0", - "version": "1.0.0", - "release_date": "2018-10-24" - } - ] - } - - -Notes -####### - -``versions`` calls ``versions()`` from `fetchcode/package_versions.py `__. - -Version information is not needed in submitted PURLs and, if included, will be removed before processing. - - ----- - - -'metadata' -- collect package metadata for a PURL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: console +- Collect package metadata for a PURL:: $ purlcli metadata --help Usage: purlcli metadata [OPTIONS] @@ -224,141 +78,11 @@ Version information is not needed in submitted PURLs and, if included, will be r --purl TEXT PackageURL or PURL. --output FILENAME Write meta output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. + --unique Return data only for unique PURLs. --help Show this message and exit. -Examples -######## - -**Submit multiple PURLs using the command line:** - -.. code-block:: none - - purlcli metadata --purl pkg:openssl/openssl@3.0.6 --purl pkg:nginx/nginx@0.8.9 --purl pkg:gnu/glibc@2.38 --output - -*Sample output:* - -.. code-block:: console - - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:openssl/openssl@3.0.6", - "pkg:nginx/nginx@0.8.9", - "pkg:gnu/glibc@2.38" - ], - "--file": null, - "--output": "" - }, - "errors": [], - "warnings": [ - "'check_existence' is not supported for 'pkg:openssl/openssl@3.0.6'", - "'pkg:nginx/nginx@0.8.9' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:gnu/glibc@2.38'" - ] - } - ], - "packages": [ - { - "purl": "pkg:openssl/openssl@3.0.6", - "type": "openssl", - "namespace": null, - "name": "openssl", - "version": "3.0.6", - "qualifiers": {}, - "subpath": null, - "primary_language": "C", - "description": null, - "release_date": "2022-10-11T12:39:09", - "parties": [], - "keywords": [], - "homepage_url": "https://www.openssl.org", - "download_url": "https://github.com/openssl/openssl/archive/refs/tags/openssl-3.0.6.tar.gz", - "api_url": "https://api.github.com/repos/openssl/openssl", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/openssl/openssl/issues", - "code_view_url": "https://github.com/openssl/openssl", - "vcs_url": "git://github.com/openssl/openssl.git", - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:gnu/glibc@2.38", - "type": "gnu", - "namespace": null, - "name": "glibc", - "version": "2.38", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": "2023-07-31T17:34:00", - "parties": [], - "keywords": [], - "homepage_url": "https://ftp.gnu.org/pub/gnu/glibc/", - "download_url": "https://ftp.gnu.org/pub/gnu/glibc/glibc-2.38.tar.gz", - "api_url": null, - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - } - - -Notes -####### - -``metadata`` calls ``info()`` from `fetchcode/package.py `__. - -The intended output for each PURL type supported by the ``metadata`` command is - -- an input PURL with a version: output the metadata for the input version -- an input PURL with no version: output a list of the metadata for all versions - -The output of the various PURL types currently supported in `fetchcode/package.py `__ varies from type to type at the moment -- the underlying functions will be updated as needed so that all produce the intended output for input PURLs with and without a version. - - ----- - - -'urls' -- collect package URLs for a PURL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: console + +- Collect package URLs for a PURL:: $ purlcli urls --help Usage: purlcli urls [OPTIONS] @@ -370,196 +94,10 @@ The output of the various PURL types currently supported in `fetchcode/package.p --purl TEXT PackageURL or PURL. --output FILENAME Write urls output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. + --unique Return data only for unique PURLs. --head Validate each URL's existence with a head request. --help Show this message and exit. -Examples -######## - -**Submit multiple PURLs using the command line:** - -.. code-block:: none - - purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output - -*Sample output:* - -.. code-block:: console - - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": [ - "pkg:npm/canonical-path@1.0.0", - "pkg:nginx/nginx@0.8.9", - "pkg:rubygems/rails@7.0.0" - ], - "--file": null, - "--output": "" - }, - "errors": [], - "warnings": [ - "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", - "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" - ] - } - ], - "packages": [ - { - "purl": "pkg:npm/canonical-path@1.0.0", - "download_url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", - "inferred_urls": [ - "https://www.npmjs.com/package/canonical-path/v/1.0.0", - "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz" - ], - "repository_download_url": null, - "repository_homepage_url": "https://www.npmjs.com/package/canonical-path/v/1.0.0" - }, - { - "purl": "pkg:rubygems/rails@7.0.0", - "download_url": "https://rubygems.org/downloads/rails-7.0.0.gem", - "inferred_urls": [ - "https://rubygems.org/gems/rails/versions/7.0.0", - "https://rubygems.org/downloads/rails-7.0.0.gem" - ], - "repository_download_url": null, - "repository_homepage_url": "https://rubygems.org/gems/rails/versions/7.0.0" - } - ] - } - - -**Include head and get requests:** - -``--head`` - -.. code-block:: none - - purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output --head - -*Sample output:* - -.. code-block:: console - - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": [ - "pkg:npm/canonical-path@1.0.0", - "pkg:nginx/nginx@0.8.9", - "pkg:rubygems/rails@7.0.0" - ], - "--file": null, - "--head": true, - "--output": "" - }, - "errors": [], - "warnings": [ - "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", - "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" - ] - } - ], - "packages": [ - { - "purl": "pkg:npm/canonical-path@1.0.0", - "download_url": { - "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", - "get_request_status_code": 200, - "head_request_status_code": 301 - }, - "inferred_urls": [ - { - "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - { - "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", - "get_request_status_code": 200, - "head_request_status_code": 301 - } - ], - "repository_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repository_homepage_url": { - "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - }, - { - "purl": "pkg:rubygems/rails@7.0.0", - "download_url": { - "url": "https://rubygems.org/downloads/rails-7.0.0.gem", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/rails/versions/7.0.0", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - { - "url": "https://rubygems.org/downloads/rails-7.0.0.gem", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - ], - "repository_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repository_homepage_url": { - "url": "https://rubygems.org/gems/rails/versions/7.0.0", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - } - ] - } - - -Notes -####### - -- None atm. - - -Testing -------- - -Run all purldb tests: - -.. code-block:: none - - make test - -Run all purlcli non-live tests (i.e., no live network calls): - -.. code-block:: none - - DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli.py - -Run all purlcli live tests (i.e., check actual API endpoints etc.) - -.. code-block:: none - - DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli_live.py --run_live_fetch - Funding ------- From 6b6a8381abff5f8f882daf8f479fb87f2a66e1ca Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 4 Jun 2024 19:48:02 +0200 Subject: [PATCH 5/9] Use latest README.rst Signed-off-by: Philippe Ombredanne --- purldb-toolkit/README.rst | 659 ++++++++++++++++++++++++++++++++++---- 1 file changed, 603 insertions(+), 56 deletions(-) diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index cb70a999..7c85e360 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -1,102 +1,649 @@ purldb-toolkit ============== -purldb-toolkit is command line utility and library to use the PurlDB, its API and various related libraries. +.. contents:: :local: + :depth: 3 -The ``purlcli`` command acts as a client to the PurlDB REST API end point(s) to expose PURL services. -It serves both as a tool, as a library and as an example on how to use the services programmatically. +purldb-toolkit is a command line utility and library to use the PurlDB, its API and various related libraries. + +purldb-toolkit exposes the ``purlcli`` command that acts as a client to the PURL libraries, +the PurlDB and MatchCode REST API and exposes various PURL-based services. + +purldb-toolkit serves as a tool, a library and an example of how to use the services programmatically. Installation ------------ +.. code-block:: console + pip install purldb-toolkit Usage ----- -Use this command to get basic help:: - - $ purlcli --help - Usage: purlcli [OPTIONS] COMMAND [ARGS]... +The purlcli command exposes multiple subcommands. Run this to command to get basic help: - Return information from a PURL. +.. code-block:: console + purlcli --help + Usage: purlcli [OPTIONS] COMMAND [ARGS]... + + Return information for a PURL or list of PURLs. + Options: --help Show this message and exit. - + Commands: - metadata Given one or more PURLs, for each PURL, return a mapping of... - urls Given one or more PURLs, for each PURL, return a list of all... - validate Check the syntax of one or more PURLs. - versions Given one or more PURLs, return a list of all known versions... + d2d Run deploy-to-devel "back2source" analysis between packages. + metadata Fetch package metadata for a PURL. + urls Return known URLs for a PURL. + validate Validate PURL syntax and existence. + versions List all known versions for a PURL. + -And the following subcommands: +The purlcli exposes the following subcommands: -- Validate a PURL:: +- validate Validate PURL syntax. +- metadata Fetch package metadata for a PURL. +- urls Return known URLs for a PURL. +- versions List all known versions for a PURL. +- d2d Run deploy-to-devel between packages. - $ purlcli validate --help - Usage: purlcli validate [OPTIONS] - Check the syntax of one or more PURLs. +Each subcommand use the same set of options:: Options: - --purl TEXT PackageURL or PURL. - --output FILENAME Write validation output as JSON to FILE. [required] - --file FILENAME Read a list of PURLs from a FILE, one per line. - --help Show this message and exit. + --purl PURL Package-URL or PURL. + --output FILE Write output as JSON to FILE. Default is to print on screen. + [default: -] + --file FILE Read a list of PURLs from a FILE, one per line. + --help Show this message and exit. -- Collect package versions for a PURL:: +``validate``: validate a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - $ purlcli versions --help - Usage: purlcli versions [OPTIONS] +This command validates a PURL in two ways: - Given one or more PURLs, return a list of all known versions for each PURL. +* validate that the PURL syntax is correct +* validate that the PURL points to an existing package querying the PurlDB and upstream + package registries as needed. - Version information is not needed in submitted PURLs and if included will be - removed before processing. - Options: - --purl TEXT PackageURL or PURL. - --output FILENAME Write versions output as JSON to FILE. [required] - --file FILENAME Read a list of PURLs from a FILE, one per line. - --help Show this message and exit. +Examples +######## +**Submit multiple PURLs using the command line:** -- Collect package metadata for a PURL:: +.. code-block:: console - $ purlcli metadata --help - Usage: purlcli metadata [OPTIONS] + purlcli validate --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --output - Given one or more PURLs, for each PURL, return a mapping of metadata fetched - from the fetchcode package.py info() function. +*Sample output:* - Options: - --purl TEXT PackageURL or PURL. - --output FILENAME Write meta output as JSON to FILE. [required] - --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. - --help Show this message and exit. +.. code-block:: json + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "validate", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:nginx/nginx@0.8.9'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo." + }, + { + "purl": "pkg:nginx/nginx@0.8.9", + "valid": true, + "exists": null, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type." + } + ] + } -- Collect package URLs for a PURL:: - $ purlcli urls --help - Usage: purlcli urls [OPTIONS] +**Submit multiple PURLs using a .txt file:** - Given one or more PURLs, for each PURL, return a list of all known URLs - fetched from the packageurl-python purl2url.py code. +.. code-block:: console - Options: - --purl TEXT PackageURL or PURL. - --output FILENAME Write urls output as JSON to FILE. [required] - --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. - --head Validate each URL's existence with a head request. - --help Show this message and exit. + purlcli validate --file --output + +*Sample input.txt:* + +.. code-block:: text + + pkg:npm/canonical-path@1.0.0 + pkg:nginx/nginx@0.8.9 + + +Details +####### + +``validate`` calls the ``validate/`` endpoint of the `purldb API `_. + +See also https://public.purldb.io/api/docs/#/validate for details. + + +---- + + +``versions``: collect package versions for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This command collects and return a list of all the known versions of a PURL by querying the PurlDB +and upstream package registries as needed. + + +This command collects and return a list of all the known versions of a PURL by querying the PurlDB +and upstream package registries as needed. + + +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: console + + purlcli versions --purl pkg:npm/canonical-path --purl pkg:nginx/nginx --output + +*Sample output:* + +.. code-block:: json + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": [ + "pkg:npm/canonical-path", + "pkg:nginx/nginx" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx' not supported with `versions` command" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@0.0.1", + "version": "0.0.1", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@0.0.2", + "version": "0.0.2", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@1.0.0", + "version": "1.0.0", + "release_date": "2018-10-24" + } + ] + } + + +Details +####### + +``versions`` calls ``versions()`` from `fetchcode/package_versions.py`. + +Version information is not needed in submitted PURLs and, if included, will be removed before processing. + + +---- + + +``metadata``: collect package metadata for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This command collects and return the metadata for the package pointed by a PURL. It does so by +querying the PurlDB and upstream package registries as needed. + +The metadata consist of all available information found in the package manifest and package registry +or repository API. + +The schema is the schema used by ScanCode Toolkit, PurlDB and all other AboutCode projects. + +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: console + + purlcli metadata --purl pkg:openssl/openssl@3.0.6 --purl pkg:nginx/nginx@0.8.9 --purl pkg:gnu/glibc@2.38 --output + +*Sample output:* + +.. code-block:: json + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": [ + "pkg:openssl/openssl@3.0.6", + "pkg:nginx/nginx@0.8.9", + "pkg:gnu/glibc@2.38" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:openssl/openssl@3.0.6'", + "'pkg:nginx/nginx@0.8.9' not supported with `metadata` command", + "'check_existence' is not supported for 'pkg:gnu/glibc@2.38'" + ] + } + ], + "packages": [ + { + "purl": "pkg:openssl/openssl@3.0.6", + "type": "openssl", + "namespace": null, + "name": "openssl", + "version": "3.0.6", + "qualifiers": {}, + "subpath": null, + "primary_language": "C", + "description": null, + "release_date": "2022-10-11T12:39:09", + "parties": [], + "keywords": [], + "homepage_url": "https://www.openssl.org", + "download_url": "https://github.com/openssl/openssl/archive/refs/tags/openssl-3.0.6.tar.gz", + "api_url": "https://api.github.com/repos/openssl/openssl", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/openssl/openssl/issues", + "code_view_url": "https://github.com/openssl/openssl", + "vcs_url": "git://github.com/openssl/openssl.git", + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:gnu/glibc@2.38", + "type": "gnu", + "namespace": null, + "name": "glibc", + "version": "2.38", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": "2023-07-31T17:34:00", + "parties": [], + "keywords": [], + "homepage_url": "https://ftp.gnu.org/pub/gnu/glibc/", + "download_url": "https://ftp.gnu.org/pub/gnu/glibc/glibc-2.38.tar.gz", + "api_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ] + } + + +Details +####### + +``metadata`` calls ``info()`` from `fetchcode/package.py`. + +The intended output for each PURL type supported by the ``metadata`` command is: + +- an input PURL with a version: output the metadata for the input version +- an input PURL without a version: output a list of the metadata for all versions + + +---- + + +``urls``: collect package URLs for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This command collects and return the known URL for a PURL. It does so by based on package type/ecosystem +conventions. It optionally also checks if the inferred URLs exists on the web. + +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: console + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output + +*Sample output:* + +.. code-block:: json + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "inferred_urls": [ + "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz" + ], + "repository_download_url": null, + "repository_homepage_url": "https://www.npmjs.com/package/canonical-path/v/1.0.0" + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "inferred_urls": [ + "https://rubygems.org/gems/rails/versions/7.0.0", + "https://rubygems.org/downloads/rails-7.0.0.gem" + ], + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/rails/versions/7.0.0" + } + ] + } + + +**Include head and get requests:** + +``--head`` + +.. code-block:: console + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output --head + +*Sample output:* + +.. code-block:: json + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--head": true, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + }, + "inferred_urls": [ + { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + "inferred_urls": [ + { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + } + ] + } + + + + +``d2d``: Run a deployed code to development code analysis +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This command runs deploy-to-devel aka. "back2source" analysis between packages. + +Its behavior depends on the number of --purl options and their values. + +- With a single PURL, run the deploy-to-devel between all the PURLs of the set of PURLs that + this PURL belongs to. + +- With two PURLs, run the deploy-to-devel between these two PURLs. The first is the "from" PURL, + and the second is the "to" PURL. The first or "from" PURL is typically the source code or version + control checkout. The second or "to" PURL is the target of a build or transformnation such as a + binary, or a source archive. + +- You can also provide two HTTP URLs instead of PURLs and use these as direct download URLs. + +This command waits for the run to complete and save results to the `output` FILE. + + + +Examples +######## + +**Run a d2d analysis between two Java JARs (source and binary)** + +You first need to install and run matchcode locally so you have the endpoint accessible. Starting +from a https://github.com/nexB/purldb/ clone:: + + git clone https://github.com/nexB/purldb + cd purldb + make dev + make envfile + SECRET_KEY="1" make postgres_matchcodeio + SECRET_KEY="1" make run_matchcodeio + +Then in another shell:: + + cd purldb + source venv/bin/activate + +Finally run the command: + +.. code-block:: console + + purlcli d2d \ + --purl https://repo1.maven.org/maven2/org/apache/htrace/htrace-core/4.0.0-incubating/htrace-core-4.0.0-incubating-sources.jar \ + --purl https://repo1.maven.org/maven2/org/apache/htrace/htrace-core/4.0.0-incubating/htrace-core-4.0.0-incubating.jar \ + --matchcode-api-url http://127.0.0.1:8002/api/ + +*Sample output:* + +Here you can see that there are over 730 resources that require review and that may be present in the +binary and not present in the sources. + +.. code-block:: json + + { + "url": "http://127.0.0.1:8002/api/d2d/5d9dbcca-48f0-4788-a356-29196f785c52/", + "uuid": "5d9dbcca-48f0-4788-a356-29196f785c52", + "created_date": "2024-06-04T16:31:24.879808Z", + "input_sources": [ + { + "uuid": "6b459edd-6b8b-473a-add7-cc79152b4d5e", + "filename": "htrace-core-4.0.0-incubating-sources.jar", + "download_url": "https://repo1.maven.org/maven2/org/apache/htrace/htrace-core/4.0.0-incubating/htrace-core-4.0.0-incubating-sources.jar#from", + "is_uploaded": false, + "tag": "from", + "size": 42766, + "is_file": true, + "exists": true + }, + { + "uuid": "bb811a08-ea8c-46b4-8720-865f068ecc0d", + "filename": "htrace-core-4.0.0-incubating.jar", + "download_url": "https://repo1.maven.org/maven2/org/apache/htrace/htrace-core/4.0.0-incubating/htrace-core-4.0.0-incubating.jar#to", + "is_uploaded": false, + "tag": "to", + "size": 1485031, + "is_file": true, + "exists": true + } + ], + "runs": [ + "8689ba05-3859-4eab-b2cf-9bec1495629f" + ], + "resource_count": 849, + "package_count": 1, + "dependency_count": 0, + "relation_count": 37, + "codebase_resources_summary": { + "ignored-directory": 56, + "mapped": 37, + "not-deployed": 1, + "requires-review": 730, + "scanned": 25 + }, + "discovered_packages_summary": { + "total": 1, + "with_missing_resources": 0, + "with_modified_resources": 0 + }, + "discovered_dependencies_summary": { + "total": 0, + "is_runtime": 0, + "is_optional": 0, + "is_resolved": 0 + }, + "codebase_relations_summary": { + "java_to_class": 34, + "sha1": 3 + }, + "codebase_resources_discrepancies": { + "total": 730 + } + } Funding From b964cac2778087f99ee6dd05520a36ebbc1984fa Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 7 Jun 2024 19:03:50 -0700 Subject: [PATCH 6/9] Remove logging and messaging #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 430 +++--------------- .../purlcli/expected_metadata_output.json | 10 +- .../data/purlcli/expected_urls_output.json | 47 +- .../purlcli/expected_urls_output_head.json | 79 +--- .../expected_urls_output_head_mock.json | 4 +- .../purlcli/expected_validate_output.json | 7 +- .../purlcli/expected_versions_output.json | 69 +-- purldb-toolkit/tests/test_purlcli.py | 384 ++-------------- purldb-toolkit/tests/test_purlcli_live.py | 338 ++------------ 9 files changed, 152 insertions(+), 1216 deletions(-) diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 28d99e7d..29f1597b 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -8,23 +8,15 @@ # import json -import logging -import os import re from importlib.metadata import version -from pathlib import Path import click import requests from fetchcode.package import info -from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS from fetchcode.package_versions import versions -from packageurl import PackageURL from packageurl.contrib import purl2url -LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") -logger = logging.getLogger(__name__) - @click.group() def purlcli(): @@ -56,8 +48,7 @@ def purlcli(): ) def get_metadata(purls, output, file): """ - Given one or more PURLs, for each PURL, return a mapping of metadata - fetched from the fetchcode package.py info() function. + Fetch package metadata for a PURL. """ check_for_duplicate_input_sources(purls, file) if file: @@ -76,39 +67,22 @@ def get_metadata_details(purls, output, file, command_name): metadata_details = {} metadata_details["headers"] = [] metadata_details["packages"] = [] - metadata_warnings = {} - deduplicated_purls, duplicate_purls = deduplicate_purls(purls) - clear_log_file() + deduplicated_purls = deduplicate_purls(purls) for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - metadata_purl_status = check_metadata_purl(purl) - if command_name == "metadata" and metadata_purl_status in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - ]: - metadata_warnings[purl] = metadata_purl_status - continue - if command_name == "metadata" and metadata_purl_status in [ - "valid_but_not_fully_supported", - "check_existence_not_supported", - ]: - metadata_warnings[purl] = metadata_purl_status metadata_collection = collect_metadata(purl) metadata_details["packages"].extend(metadata_collection) metadata_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, - duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - purl_warnings=metadata_warnings, ) + return metadata_details @@ -124,45 +98,8 @@ def collect_metadata(purl): release_detail = release.to_dict() release_detail.move_to_end("purl", last=False) collected_metadata.append(release_detail) - return collected_metadata - -def check_metadata_purl(purl): - """ - Return a variable identifying the warning if (1) the input PURL is invalid, - (2) its type is not supported by `metadata` or (3) its existence was not - validated (e.g., "does not exist in the upstream repo"). This will be - reported by construct_headers() in the `warnings` field of the `header` - section of the JSON object returned by the `metadata` command. - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - elif check_validation["valid"] == False: - return "not_valid" - - # This is manually constructed from a visual inspection of fetchcode/package.py. - metadata_supported_ecosystems = [ - "bitbucket", - "cargo", - "generic", - "github", - "gnu", - "npm", - "openssl", - "pypi", - "rubygems", - # NOTE: cocoapods support added subject to fetchcode/package.py PR approval and new release. - "cocoapods", - ] - metadata_purl = PackageURL.from_string(purl) - - if metadata_purl.type not in metadata_supported_ecosystems: - return "valid_but_not_supported" - elif check_validation["exists"] == False: - return "not_in_upstream_repo" - elif check_validation["exists"] == None: - return "check_existence_not_supported" + return collected_metadata def deduplicate_purls(purls): @@ -172,34 +109,21 @@ def deduplicate_purls(purls): """ reviewed = set() deduplicated_purls = [] - duplicate_purls = [] for purl in purls: purl = purl.strip() if purl not in reviewed: reviewed.add(purl) deduplicated_purls.append(purl) - else: - duplicate_purls.append(purl) - return deduplicated_purls, duplicate_purls - -def read_log_file(log_file_path): - log_file = log_file_path - if log_file.is_file(): - with open(log_file_path, "r") as log_file: - return log_file.readlines() - else: - return [] + return deduplicated_purls def construct_headers( deduplicated_purls=None, - duplicate_purls=None, output=None, file=None, command_name=None, head=None, - purl_warnings=None, ): """ Return a list comprising the `headers` content of the dictionary output. @@ -232,42 +156,10 @@ def construct_headers( options["--output"] = output.name headers_content["options"] = options - if command_name in ["metadata", "urls", "validate", "versions"]: - if duplicate_purls: - for duplicate in duplicate_purls: - logger.warning(f"Duplicate input PURL removed: {duplicate}") - - for purl in deduplicated_purls: - if not purl: - continue - warning_text = { - "error_fetching_purl": f"'error fetching {purl}'", - "validation_error": f"'{purl}' encountered a validation error", - "not_valid": f"'{purl}' not valid", - "valid_but_not_supported": f"'{purl}' not supported with `{command_name}` command", - "valid_but_not_fully_supported": f"'{purl}' not fully supported with `urls` command", - "not_in_upstream_repo": f"'{purl}' does not exist in the upstream repo", - "check_existence_not_supported": f"'check_existence' is not supported for '{purl}'", - } - if command_name in ["metadata", "urls", "validate", "versions"]: - purl_warning = purl_warnings.get(purl, None) - if purl_warning: - warning = warning_text[purl_warning] - logger.warning(warning) - continue - - log_file = Path(LOG_FILE_LOCATION) - log_file_contents = read_log_file(log_file) - if log_file_contents: - for line in log_file_contents: - if line.startswith("ERROR"): - errors.append(line[8:-1]) - elif line.startswith("WARNING"): - warnings.append(line[10:-1]) - headers_content["errors"] = errors headers_content["warnings"] = warnings headers.append(headers_content) + return headers @@ -300,8 +192,11 @@ def construct_headers( ) def get_urls(purls, output, file, head): """ - Given one or more PURLs, for each PURL, return a list of all known URLs - fetched from the packageurl-python purl2url.py code. + Return known URLs for a PURL. + + This includes the "download_url" which is the standard download URL, the "repo_download_url" + which is the download URL provided by the package repository, the "repo_url" which is the URL of + this package on the package repository. """ check_for_duplicate_input_sources(purls, file) if file: @@ -320,9 +215,7 @@ def get_urls_details(purls, output, file, head, command_name): urls_details = {} urls_details["headers"] = [] urls_details["packages"] = [] - urls_warnings = {} - deduplicated_purls, duplicate_purls = deduplicate_purls(purls) - clear_log_file() + deduplicated_purls = deduplicate_purls(purls) for purl in deduplicated_purls: url_detail = {} @@ -330,22 +223,6 @@ def get_urls_details(purls, output, file, head, command_name): purl = purl.strip() if not purl: continue - urls_purl_status = check_urls_purl(purl) - if command_name == "urls" and urls_purl_status in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - ]: - urls_warnings[purl] = urls_purl_status - continue - if command_name == "urls" and urls_purl_status in [ - "valid_but_not_fully_supported", - "check_existence_not_supported", - ]: - urls_warnings[purl] = urls_purl_status - - url_purl = PackageURL.from_string(purl) url_detail["download_url"] = purl2url.get_download_url(purl) if head: @@ -393,23 +270,18 @@ def get_urls_details(purls, output, file, head, command_name): urls_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, - duplicate_purls=duplicate_purls, output=output, file=file, head=head, command_name=command_name, - purl_warnings=urls_warnings, ) + return urls_details def make_head_request(url_detail): """ - Make a head request and get request and return a dictionary containing - status code data for the incoming PURL URL. This returns both get and - head request status code data so the user can evaluate -- requests.get() - and requests.head() sometimes return different status codes and sometimes - return inaccurate codes, e.g., a 404 when the URL actually exists. + Return both get and head request status code data for each URL. """ if url_detail is None: return {"get_request": "N/A", "head_request": "N/A"} @@ -417,76 +289,13 @@ def make_head_request(url_detail): get_request_status_code = get_response.status_code head_response = requests.head(url_detail) head_request_status_code = head_response.status_code + return { "get_request": get_request_status_code, "head_request": head_request_status_code, } -def check_urls_purl(purl): - """ - If applicable, return a variable indicating that the input PURL is invalid, - or its type is not supported (or not fully supported) by `urls`, or it - does not exist in the upstream repo. - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - results = check_validation - if results["valid"] == False: - return "not_valid" - - # Both of these lists are manually constructed from a visual inspection of - # packageurl-python/src/packageurl/contrib/purl2url.py. - # This list applies to the purl2url.py `repo_url` section: - urls_supported_ecosystems_repo_url = [ - "bitbucket", - "cargo", - # NOTE: Temp for cocoapods dev work in purl2url. Keep in the list uncommented -- I still need to respond to Tushar's comments but the real code work is done, now supported for repo_url. - "cocoapods", - "gem", - "github", - "gitlab", - "golang", - "hackage", - "npm", - "nuget", - "pypi", - "rubygems", - ] - # This list applies to the purl2url.py `download_url` section: - urls_supported_ecosystems_download_url = [ - "bitbucket", - "cargo", - "gem", - "github", - "gitlab", - "hackage", - "npm", - "nuget", - "rubygems", - ] - urls_purl = PackageURL.from_string(purl) - - if ( - urls_purl.type not in urls_supported_ecosystems_repo_url - and urls_purl.type not in urls_supported_ecosystems_download_url - ): - return "valid_but_not_supported" - if results["exists"] == False: - return "not_in_upstream_repo" - if ( - urls_purl.type in urls_supported_ecosystems_repo_url - and urls_purl.type not in urls_supported_ecosystems_download_url - ) or ( - urls_purl.type not in urls_supported_ecosystems_repo_url - and urls_purl.type in urls_supported_ecosystems_download_url - ): - return "valid_but_not_fully_supported" - if results["exists"] == None: - return "check_existence_not_supported" - - @purlcli.command(name="validate") @click.option( "--purl", @@ -510,7 +319,9 @@ def check_urls_purl(purl): ) def validate(purls, output, file): """ - Check the syntax and upstream repo status of one or more PURLs. + Validate PURL syntax and existence. + + Check that the syntax of a PURL is correct. Check that the PURL exists using the PurlDB. """ check_for_duplicate_input_sources(purls, file) if file: @@ -528,105 +339,41 @@ def get_validate_details(purls, output, file, command_name): """ validate_details = {} validate_details["headers"] = [] - validate_warnings = {} - deduplicated_purls, duplicate_purls = deduplicate_purls(purls) validate_details["packages"] = [] - clear_log_file() + deduplicated_purls = deduplicate_purls(purls) for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - validated_purl_status = check_validate_purl(purl) - if command_name == "validate" and validated_purl_status in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - "check_existence_not_supported", - ]: - validate_warnings[purl] = validated_purl_status - if validated_purl_status: - # Move the `purl` key to the top. - original_validate_purl = validate_purl(purl) - reordered_validate_purl = { - "purl": original_validate_purl.pop("purl"), - **original_validate_purl, - } - validate_details["packages"].append(reordered_validate_purl) + + original_validate_purl = validate_purl(purl) + reordered_validate_purl = { + "purl": original_validate_purl.pop("purl"), + **original_validate_purl, + } + validate_details["packages"].append(reordered_validate_purl) validate_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, - duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - purl_warnings=validate_warnings, ) - return validate_details - -def check_validate_purl(purl): - """ - As applicable, return a variable indicating that the input PURL is - valid/invalid or does not exist in the upstream repo. - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - elif check_validation["valid"] == False: - return "not_valid" - elif check_validation["exists"] == False: - return "not_in_upstream_repo" - elif check_validation["exists"] == True: - return check_validation - elif check_validation["exists"] == None: - return "check_existence_not_supported" + return validate_details def validate_purl(purl): """ Return a JSON object containing data from the PurlDB `validate` endpoint regarding the validity of the input PURL. - - Based on packagedb.package_managers VERSION_API_CLASSES_BY_PACKAGE_TYPE - and packagedb/api.py class PurlValidateViewSet(viewsets.ViewSet) - -- and supported by testing the command -- it appears that the `validate` - command `check_existence` parameter supports the following PURL types: - - "cargo", - "composer", - "deb", - "gem", - "golang", - "hex", - "maven", - "npm", - "nuget", - "pypi", """ - logging.basicConfig( - level=logging.WARN, - format="%(levelname)s - %(message)s", - filename=LOG_FILE_LOCATION, - filemode="w", - ) - api_query = "https://public.purldb.io/api/validate/" request_body = {"purl": purl, "check_existence": True} + response = requests.get(api_query, params=request_body).json() - try: - response = requests.get(api_query, params=request_body).json() - except json.decoder.JSONDecodeError as e: - logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - except Exception as e: - logger.error(f"'validate' endpoint error for '{purl}': {e}") - else: - if response is None: - logger.error( - f"'{purl}' -- response.status_code for None = {response.status_code}" - ) - return response + return response @purlcli.command(name="versions") @@ -652,7 +399,7 @@ def validate_purl(purl): ) def get_versions(purls, output, file): """ - Given one or more PURLs, return a list of all known versions for each PURL. + List all known versions for a PURL. """ check_for_duplicate_input_sources(purls, file) if file: @@ -671,116 +418,55 @@ def get_versions_details(purls, output, file, command_name): versions_details = {} versions_details["headers"] = [] versions_details["packages"] = [] - versions_warnings = {} - deduplicated_purls, duplicate_purls = deduplicate_purls(purls) - clear_log_file() - for purl in deduplicated_purls: - purl = purl.strip() - if not purl: + raw_purls = [] + for input_purl in purls: + raw_purl = re.split("[@,]+", input_purl)[0] + raw_purls.append(raw_purl) + + deduplicated_purls = deduplicate_purls(raw_purls) + for deduplicated_purl in deduplicated_purls: + deduplicated_purl = deduplicated_purl.strip() + if not deduplicated_purl: continue purl_data = {} - purl_data["purl"] = purl - versions_purl_status = check_versions_purl(purl) - if command_name == "versions" and versions_purl_status in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - ]: - versions_warnings[purl] = versions_purl_status - continue - if command_name == "versions" and versions_purl_status in [ - "valid_but_not_fully_supported", - "check_existence_not_supported", - ]: - versions_warnings[purl] = versions_purl_status - version_collection = collect_versions(purl) + purl_data["purl"] = deduplicated_purl + + version_collection = collect_versions(deduplicated_purl) versions_details["packages"].extend(version_collection) versions_details["headers"] = construct_headers( - deduplicated_purls=deduplicated_purls, - duplicate_purls=duplicate_purls, + deduplicated_purls=purls, output=output, file=file, command_name=command_name, - purl_warnings=versions_warnings, ) + return versions_details def collect_versions(purl): """ - Return a list of version objects collected from fetchcode/package_versions.py. - - We use `versions()` from fetchcode/package_versions.py, which keeps the - version (if any) of the input PURL in its output, so - "pkg:pypi/fetchcode@0.3.0" is returned as "pkg:pypi/fetchcode@0.3.0@0.1.0", - "pkg:pypi/fetchcode@0.3.0@0.2.0" etc. Thus, we remove any string starting - with `@` first. + Return a list of version objects for each input PURL. """ collected_versions = [] for package_version in list(versions(purl)): purl_version_data = {} purl_version = package_version.value - raw_purl = re.split("[@,]+", purl)[0] - nested_purl = raw_purl + "@" + f"{purl_version}" - pkg_ver_release_date = package_version.release_date - pkg_ver_release_date_no_time = pkg_ver_release_date.date() - - purl_version_data["purl"] = nested_purl + purl_version_data["purl"] = purl purl_version_data["version"] = f"{purl_version}" - purl_version_data["release_date"] = f"{pkg_ver_release_date_no_time}" - collected_versions.append(purl_version_data) - return collected_versions + pkg_ver_release_date_no_time = None + if package_version.release_date: + pkg_ver_release_date = package_version.release_date + pkg_ver_release_date_no_time = str(pkg_ver_release_date.date()) + purl_version_data["release_date"] = f"{pkg_ver_release_date_no_time}" -def check_versions_purl(purl): - """ - Return a variable identifying the message for printing to the console by - get_versions_details() if (1) the input PURL is invalid, (2) its type is not - supported by `versions` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). This message will also be reported - by construct_headers() in the `warnings` field of the `header` section of - the JSON object returned by the `versions` command. - - Note for dev purposes: SUPPORTED_ECOSYSTEMS (imported from - fetchcode.package_versions) comprises the following types: - [ - "cargo", - "composer", - "conan", - "deb", - "gem", - "github", - "golang", - "hex", - "maven", - "npm", - "nuget", - "pypi", - ] - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - elif check_validation["valid"] == False: - return "not_valid" - - supported = SUPPORTED_ECOSYSTEMS - versions_purl = PackageURL.from_string(purl) - if versions_purl.type not in supported: - return "valid_but_not_supported" - elif check_validation["exists"] == False: - return "not_in_upstream_repo" - elif check_validation["exists"] == None: - return "check_existence_not_supported" - # This handles the conflict between the `validate`` endpoint (treats - # both "pkg:deb/debian/2ping" and "pkg:deb/2ping" as valid) and - # fetchcode.package_versions versions() (returns None for "pkg:deb/2ping"). - elif versions(purl) is None: - return "valid_but_not_supported" + purl_version_data["release_date"] = pkg_ver_release_date_no_time + collected_versions.append(purl_version_data) + + return collected_versions def check_for_duplicate_input_sources(purls, file): @@ -790,11 +476,5 @@ def check_for_duplicate_input_sources(purls, file): raise click.UsageError("Use either purls or file.") -def clear_log_file(): - log_file = Path(LOG_FILE_LOCATION) - with open(log_file, "w"): - pass - - if __name__ == "__main__": purlcli() diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 32df4492..794022cb 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -12,21 +12,13 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", "pkg:rubygems/rails" ], "--file": null, "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] + "warnings": [] } ], "packages": [ diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index c2bf50d7..4a940958 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -8,47 +8,20 @@ "--purl": [ "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", "pkg:pypi/dejacode", "pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", "pkg:cargo/banquo", "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", "pkg:gem/bundler-sass", "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", "pkg:nuget/auth0-aspnet@1.1.0" ], "--file": null, "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command", - "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", - "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] + "warnings": [] } ], "packages": [ @@ -97,24 +70,6 @@ "repository_download_url": null, "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, - { - "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": null, - "inferred_urls": [ - "https://pypi.org/project/dejacode/5.0.0/" - ], - "repository_download_url": null, - "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" - }, - { - "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": null, - "inferred_urls": [ - "https://pypi.org/project/dejacode/5.0.0/" - ], - "repository_download_url": null, - "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" - }, { "purl": "pkg:cargo/banquo", "download_url": null, diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index 2e8de8c1..0237229f 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -8,23 +8,13 @@ "--purl": [ "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", "pkg:pypi/dejacode", "pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", "pkg:cargo/banquo", "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", "pkg:gem/bundler-sass", "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", "pkg:nuget/auth0-aspnet@1.1.0" ], "--file": null, @@ -32,24 +22,7 @@ "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command", - "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", - "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] + "warnings": [] } ], "packages": [ @@ -178,56 +151,6 @@ "head_request_status_code": 200 } }, - { - "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - ], - "repository_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repository_homepage_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - }, - { - "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - ], - "repository_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repository_homepage_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - }, { "purl": "pkg:cargo/banquo", "download_url": { diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 7de378fd..2c157d29 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -13,9 +13,7 @@ "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ] + "warnings": [] } ], "packages": [ diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json index d1f74eba..de84f52d 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json @@ -20,12 +20,7 @@ "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'check_existence' is not supported for 'pkg:nginx/nginx'", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] + "warnings": [] } ], "packages": [ diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json index e9c6abf8..428979d9 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json @@ -12,128 +12,93 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:nginx/nginx", "pkg:hex/coherence@0.1.0" ], "--file": null, "--output": "" }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `versions` command" - ] + "warnings": [] } ], "packages": [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25" }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14" }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18" }, { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18" - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18" - }, - { - "purl": "pkg:cargo/banquo@0.1.0", + "purl": "pkg:cargo/banquo", "version": "0.1.0", "release_date": "2024-02-07" }, { - "purl": "pkg:hex/coherence@0.8.0", + "purl": "pkg:hex/coherence", "version": "0.8.0", "release_date": "2023-09-22" }, { - "purl": "pkg:hex/coherence@0.5.2", + "purl": "pkg:hex/coherence", "version": "0.5.2", "release_date": "2018-09-03" }, { - "purl": "pkg:hex/coherence@0.5.1", + "purl": "pkg:hex/coherence", "version": "0.5.1", "release_date": "2018-08-28" }, { - "purl": "pkg:hex/coherence@0.5.0", + "purl": "pkg:hex/coherence", "version": "0.5.0", "release_date": "2017-08-02" }, { - "purl": "pkg:hex/coherence@0.4.0", + "purl": "pkg:hex/coherence", "version": "0.4.0", "release_date": "2017-07-03" }, { - "purl": "pkg:hex/coherence@0.3.1", + "purl": "pkg:hex/coherence", "version": "0.3.1", "release_date": "2016-11-27" }, { - "purl": "pkg:hex/coherence@0.3.0", + "purl": "pkg:hex/coherence", "version": "0.3.0", "release_date": "2016-08-28" }, { - "purl": "pkg:hex/coherence@0.2.0", + "purl": "pkg:hex/coherence", "version": "0.2.0", "release_date": "2016-07-30" }, { - "purl": "pkg:hex/coherence@0.1.3", + "purl": "pkg:hex/coherence", "version": "0.1.3", "release_date": "2016-07-19" }, { - "purl": "pkg:hex/coherence@0.1.2", + "purl": "pkg:hex/coherence", "version": "0.1.2", "release_date": "2016-07-12" }, { - "purl": "pkg:hex/coherence@0.1.1", + "purl": "pkg:hex/coherence", "version": "0.1.1", "release_date": "2016-07-11" }, { - "purl": "pkg:hex/coherence@0.1.0", + "purl": "pkg:hex/coherence", "version": "0.1.0", "release_date": "2016-07-11" } diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 43def355..f4035125 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -55,9 +55,7 @@ def test_metadata_cli_no_input_sources(self): assert result.exit_code == 2 @mock.patch("purldb_toolkit.purlcli.collect_metadata") - @mock.patch("purldb_toolkit.purlcli.check_metadata_purl") - def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata): - + def test_metadata_details(self, mock_collect_metadata): mock_collect_metadata.return_value = [ OrderedDict( [ @@ -222,8 +220,6 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ), ] - mock_check_metadata_purl.return_value = None - expected_data = { "headers": [ { @@ -421,88 +417,6 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) assert purl_metadata_data == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_metadata_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - } - input_purl = "pkg:rubygems/bundler-sass" - expected = "check_existence_not_supported" - purl_metadata = purlcli.check_metadata_purl(input_purl) - - assert purl_metadata == expected - - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_metadata_purl_multiple(self, mock_validate_purl): - mock_validate_purl.side_effect = [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - }, - { - "valid": True, - "exists": False, - "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", - "purl": "pkg:pypi/zzzzz", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "pkg:pypi/?fetchcode", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "zzzzz", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:maven/axis/axis@1.0", - }, - ] - - input_purls_and_expected_states = [ - ["pkg:pypi/fetchcode", None], - ["pkg:gem/bundler-sass", "valid_but_not_supported"], - ["pkg:rubygems/bundler-sass", "check_existence_not_supported"], - ["pkg:nginx/nginx", "valid_but_not_supported"], - ["pkg:pypi/zzzzz", "not_in_upstream_repo"], - ["pkg:pypi/?fetchcode", "not_valid"], - ["zzzzz", "not_valid"], - ["pkg:maven/axis/axis@1.0", "valid_but_not_supported"], - ] - - for input_purl, expected_state in input_purls_and_expected_states: - purl_metadata = purlcli.check_metadata_purl(input_purl) - assert purl_metadata == expected_state - def test_deduplicate_purls(self): input_purls = [ "pkg:pypi/fetchcode@0.1.0", @@ -515,14 +429,7 @@ def test_deduplicate_purls(self): ] actual_output = purlcli.deduplicate_purls(input_purls) expected_output = ( - ["pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0"], - [ - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], + ["pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0"] ) assert actual_output == expected_output @@ -558,37 +465,19 @@ def test_deduplicate_purls(self): }, "tool_name": "purlcli", "tool_version": "0.1.0", - "warnings": [ - "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", - "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", - "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", - "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", - "Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0", - ], + "warnings": [], } ], ), ], ) - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_deduplicate_purls_construct_headers( - self, mock_read_log_file, test_input, expected - ): - mock_read_log_file.return_value = [ - "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", - "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", - "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", - "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", - "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0\n", - ] - + def test_deduplicate_purls_construct_headers(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - purl_warnings={}, ) cli_test_utils.streamline_headers(expected) @@ -622,27 +511,19 @@ def test_deduplicate_purls_construct_headers( }, "tool_name": "purlcli", "tool_version": "0.1.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], + "warnings": [], } ], ), ], ) - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_construct_headers(self, mock_read_log_file, test_input, expected): - mock_read_log_file.return_value = [ - "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", - ] - + def test_construct_headers(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) cli_test_utils.streamline_headers(expected) @@ -652,9 +533,8 @@ def test_construct_headers(self, mock_read_log_file, test_input, expected): class TestPURLCLI_urls(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.make_head_request") - def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): + def test_urls_cli_head(self, mock_make_head_request): """ Test the `urls` command with actual and expected JSON output files. """ @@ -671,10 +551,6 @@ def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): {"head_request": 200}, ] - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head_mock.json" ) @@ -763,15 +639,7 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @mock.patch("purldb_toolkit.purlcli.read_log_file") - @mock.patch("purldb_toolkit.purlcli.check_urls_purl") - def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): - mock_check_urls_purl.return_value = "valid_but_not_fully_supported" - - mock_read_log_file.return_value = [ - "WARNING = 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", - ] - + def test_urls_details(self): expected_data = { "headers": [ { @@ -784,9 +652,7 @@ def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): "--output": "", }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ], + "warnings": [], } ], "packages": [ @@ -816,21 +682,6 @@ def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): assert purl_urls == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_urls_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - } - - input_purl = "pkg:pypi/fetchcode" - expected = "valid_but_not_fully_supported" - purl_urls = purlcli.check_urls_purl(input_purl) - - assert purl_urls == expected - @mock.patch("requests.get") @mock.patch("requests.head") def test_validate_purl_mock_requests_get_and_head( @@ -873,72 +724,48 @@ def mock_requests_get_return_func(): class TestPURLCLI_versions(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") - @mock.patch("purldb_toolkit.purlcli.collect_versions") - @mock.patch("purldb_toolkit.purlcli.check_versions_purl") - def test_versions_details_multiple( - self, mock_check_versions_purl, mock_collect_versions, mock_read_log_file - ): - - mock_check_versions_purl.side_effect = [ - None, - None, - "valid_but_not_supported", - "valid_but_not_supported", - None, - "not_valid", - ] + @mock.patch("purldb_toolkit.purlcli.collect_versions") + def test_versions_details_multiple(self, mock_collect_versions): mock_collect_versions.side_effect = [ [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18", }, ], [ { - "purl": "pkg:gem/bundler-sass@0.1.2", + "purl": "pkg:gem/bundler-sass", "version": "0.1.2", "release_date": "2013-12-11", } ], [ { - "purl": "pkg:cargo/socksprox@0.1.1", + "purl": "pkg:cargo/socksprox", "release_date": "2024-02-07", "version": "0.1.1", }, { - "purl": "pkg:cargo/socksprox@0.1.0", + "purl": "pkg:cargo/socksprox", "release_date": "2024-02-07", "version": "0.1.0", }, ], ] - mock_read_log_file.side_effect = [ - [], - [], - [ - "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", - ], - ["WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n"], - [], - ["WARNING - 'pkg:pypi/?fetchcode' not valid\n"], - ] - input_purls_and_expected_purl_data = [ [ ["pkg:pypi/fetchcode"], @@ -959,17 +786,17 @@ def test_versions_details_multiple( ], "packages": [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18", }, @@ -995,57 +822,13 @@ def test_versions_details_multiple( ], "packages": [ { - "purl": "pkg:gem/bundler-sass@0.1.2", + "purl": "pkg:gem/bundler-sass", "version": "0.1.2", "release_date": "2013-12-11", } ], }, ], - [ - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ], - [ - ["pkg:nginx/nginx"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:nginx/nginx"], - "--file": None, - "--output": "", - }, - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ], [ ["pkg:cargo/socksprox"], { @@ -1065,38 +848,18 @@ def test_versions_details_multiple( ], "packages": [ { - "purl": "pkg:cargo/socksprox@0.1.1", + "purl": "pkg:cargo/socksprox", "version": "0.1.1", "release_date": "2024-02-07", }, { - "purl": "pkg:cargo/socksprox@0.1.0", + "purl": "pkg:cargo/socksprox", "version": "0.1.0", "release_date": "2024-02-07", }, ], }, ], - [ - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/?fetchcode"], - "--file": None, - "--output": "", - }, - "errors": [], - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ], ] output = "" @@ -1114,33 +877,25 @@ def test_versions_details_multiple( assert purl_versions_data == expected_data @mock.patch("purldb_toolkit.purlcli.collect_versions") - @mock.patch("purldb_toolkit.purlcli.check_versions_purl") - def test_versions_details( - self, - mock_check_versions_purl, - mock_collect_versions, - ): - + def test_versions_details(self, mock_collect_versions): mock_collect_versions.return_value = [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18", }, ] - mock_check_versions_purl.return_value = None - expected_data = { "headers": [ { @@ -1158,17 +913,17 @@ def test_versions_details( ], "packages": [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18", }, @@ -1189,85 +944,6 @@ def test_versions_details( ) assert purl_versions_data == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_versions_purl_multiple(self, mock_validate_purl): - mock_validate_purl.side_effect = [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - }, - { - "valid": True, - "exists": False, - "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", - "purl": "pkg:pypi/zzzzz", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "pkg:pypi/?fetchcode", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "zzzzz", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:maven/axis/axis@1.0", - }, - ] - input_purls_and_expected_states = [ - ["pkg:pypi/fetchcode", None], - ["pkg:gem/bundler-sass", None], - ["pkg:rubygems/bundler-sass", "valid_but_not_supported"], - ["pkg:nginx/nginx", "valid_but_not_supported"], - ["pkg:pypi/zzzzz", "not_in_upstream_repo"], - ["pkg:pypi/?fetchcode", "not_valid"], - ["zzzzz", "not_valid"], - ["pkg:maven/axis/axis@1.0", None], - ] - for input_purl, expected_state in input_purls_and_expected_states: - purl_versions = purlcli.check_versions_purl(input_purl) - assert purl_versions == expected_state - - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_versions_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - } - input_purl = "pkg:rubygems/bundler-sass" - purl_versions = purlcli.check_versions_purl(input_purl) - expected = "valid_but_not_supported" - assert purl_versions == expected - def streamline_metadata_packages(packages): """ diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 372b7bab..53765375 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -25,8 +25,7 @@ class TestPURLCLI_metadata(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_metadata_cli(self, mock_read_log_file): + def test_metadata_cli(self): """ Test the `metadata` command with actual and expected JSON output files. @@ -34,14 +33,6 @@ def test_metadata_cli(self, mock_read_log_file): because the `--output` values (paths) differ due to the use of temporary files, and therefore we test a list of relevant key-value pairs. """ - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:nginx/nginx' not supported with `metadata` command\n", - "WARNING - 'pkg:gem/rails' not supported with `metadata` command\n", - "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_metadata_output.json" ) @@ -60,10 +51,6 @@ def test_metadata_cli(self, mock_read_log_file): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", "pkg:rubygems/rails", "--output", actual_result_file, @@ -175,12 +162,16 @@ def test_metadata_details(self): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), ("parties", []), ("keywords", []), ("homepage_url", "https://github.com/nexB/fetchcode"), + ("download_url", None), ("api_url", "https://pypi.org/pypi/fetchcode/json"), ("size", None), ("sha1", None), @@ -188,6 +179,7 @@ def test_metadata_details(self): ("sha256", None), ("sha512", None), ("bug_tracking_url", None), + ("code_view_url", None), ("vcs_url", None), ("copyright", None), ("license_expression", None), @@ -197,9 +189,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -211,12 +200,19 @@ def test_metadata_details(self): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), ("parties", []), ("keywords", []), ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + ), ("api_url", "https://pypi.org/pypi/fetchcode/json"), ("size", None), ("sha1", None), @@ -224,6 +220,7 @@ def test_metadata_details(self): ("sha256", None), ("sha512", None), ("bug_tracking_url", None), + ("code_view_url", None), ("vcs_url", None), ("copyright", None), ("license_expression", None), @@ -233,9 +230,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -247,12 +241,19 @@ def test_metadata_details(self): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), ("parties", []), ("keywords", []), ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + ), ("api_url", "https://pypi.org/pypi/fetchcode/json"), ("size", None), ("sha1", None), @@ -260,6 +261,7 @@ def test_metadata_details(self): ("sha256", None), ("sha512", None), ("bug_tracking_url", None), + ("code_view_url", None), ("vcs_url", None), ("copyright", None), ("license_expression", None), @@ -269,9 +271,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -283,12 +282,19 @@ def test_metadata_details(self): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), ("parties", []), ("keywords", []), ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + ), ("api_url", "https://pypi.org/pypi/fetchcode/json"), ("size", None), ("sha1", None), @@ -296,6 +302,7 @@ def test_metadata_details(self): ("sha256", None), ("sha512", None), ("bug_tracking_url", None), + ("code_view_url", None), ("vcs_url", None), ("copyright", None), ("license_expression", None), @@ -305,9 +312,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), ], @@ -334,43 +338,6 @@ def test_metadata_details(self): assert purl_metadata_data["headers"] == expected_data["headers"] compare_packages(expected_data, purl_metadata_data) - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - None, - ), - ( - ["pkg:gem/bundler-sass"], - "valid_but_not_supported", - ), - ( - ["pkg:rubygems/bundler-sass"], - "check_existence_not_supported", - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_metadata_purl(self, test_input, expected): - purl_metadata = purlcli.check_metadata_purl(test_input[0]) - assert purl_metadata == expected - @pytest.mark.parametrize( "test_input,expected", [ @@ -397,27 +364,19 @@ def test_check_metadata_purl(self, test_input, expected): }, "tool_name": "purlcli", "tool_version": "0.2.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], + "warnings": [], } ], ), ], ) - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_construct_headers(self, mock_read_log_file, test_input, expected): - mock_read_log_file.return_value = [ - "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", - ] - + def test_construct_headers(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -426,30 +385,10 @@ def test_construct_headers(self, mock_read_log_file, test_input, expected): class TestPURLCLI_urls(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_urls_cli(self, mock_read_log_file): + def test_urls_cli(self): """ Test the `urls` command with actual and expected JSON output files. """ - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", - "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", - "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", - "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", - "WARNING - 'abcdefg' not valid\n", - "WARNING - 'pkg/abc' not valid\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output.json" ) @@ -460,40 +399,20 @@ def test_urls_cli(self, mock_read_log_file): "--purl", "pkg:pypi/fetchcode@0.3.0", "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", "pkg:pypi/dejacode", "--purl", "pkg:pypi/dejacode@5.0.0", "--purl", "pkg:pypi/dejacode@5.0.0?os=windows", "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", "pkg:cargo/banquo", "--purl", "pkg:cargo/socksprox", "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", "pkg:gem/bundler-sass", "--purl", "pkg:rubygems/bundler-sass", "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", "pkg:nuget/auth0-aspnet@1.1.0", "--output", actual_result_file, @@ -541,30 +460,10 @@ def test_urls_cli(self, mock_read_log_file): for output, expected in result_objects: assert output == expected - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_urls_cli_head(self, mock_read_log_file): + def test_urls_cli_head(self): """ Test the `urls` command with actual and expected JSON output files. """ - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", - "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", - "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", - "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", - "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", - "WARNING - 'abcdefg' not valid\n", - "WARNING - 'pkg/abc' not valid\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head.json" ) @@ -575,40 +474,20 @@ def test_urls_cli_head(self, mock_read_log_file): "--purl", "pkg:pypi/fetchcode@0.3.0", "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", "pkg:pypi/dejacode", "--purl", "pkg:pypi/dejacode@5.0.0", "--purl", "pkg:pypi/dejacode@5.0.0?os=windows", "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", "pkg:cargo/banquo", "--purl", "pkg:cargo/socksprox", "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", "pkg:gem/bundler-sass", "--purl", "pkg:rubygems/bundler-sass", "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", "pkg:nuget/auth0-aspnet@1.1.0", "--head", "--output", @@ -691,13 +570,7 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_urls_details(self, mock_read_log_file): - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", - "WARNING - 'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'\n", - ] - + def test_urls_details(self): input_purls = [ "pkg:pypi/fetchcode@0.3.0", "pkg:gem/bundler@2.3.23", @@ -732,10 +605,7 @@ def test_urls_details(self, mock_read_log_file): "--output": "", }, "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", - "'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'", - ], + "warnings": [], } ], "packages": [ @@ -773,43 +643,6 @@ def test_urls_details(self, mock_read_log_file): assert purl_urls_data == expected_data - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - "valid_but_not_fully_supported", - ), - ( - ["pkg:gem/bundler-sass"], - None, - ), - ( - ["pkg:rubygems/bundler-sass"], - "check_existence_not_supported", - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_urls_purl(self, test_input, expected): - purl_urls = purlcli.check_urls_purl(test_input[0]) - assert purl_urls == expected - @pytest.mark.parametrize( "test_input,expected", [ @@ -842,18 +675,10 @@ def test_make_head_request(self, test_input, expected): class TestPURLCLI_validate(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_validate_cli(self, mock_read_log_file): + def test_validate_cli(self): """ Test the `validate` command with actual and expected JSON output files. """ - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", - "WARNING - 'check_existence' is not supported for 'pkg:nginx/nginx'\n", - "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_validate_output.json" ) @@ -1073,17 +898,10 @@ def test_validate_purl_strip(self, test_input, expected): class TestPURLCLI_versions(object): - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_versions_cli(self, mock_read_log_file): + def test_versions_cli(self): """ Test the `versions` command with actual and expected JSON output files. """ - mock_read_log_file.return_value = [ - "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", - "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", - ] - expected_result_file = test_env.get_test_loc( "purlcli/expected_versions_output.json" ) @@ -1102,8 +920,6 @@ def test_versions_cli(self, mock_read_log_file): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:nginx/nginx", - "--purl", "pkg:hex/coherence@0.1.0", "--output", actual_result_file, @@ -1159,11 +975,7 @@ def test_versions_cli(self, mock_read_log_file): [ "pkg:pypi/fetchcode", "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:nginx/nginx", "pkg:pypi/zzzzz", - "pkg:pypi/?fetchcode", - "zzzzz", ], { "headers": [ @@ -1175,48 +987,38 @@ def test_versions_cli(self, mock_read_log_file): "--purl": [ "pkg:pypi/fetchcode", "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:nginx/nginx", "pkg:pypi/zzzzz", - "pkg:pypi/?fetchcode", - "zzzzz", ], "--file": None, "--output": "", }, "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command", - "'pkg:nginx/nginx' not supported with `versions` command", - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - "'pkg:pypi/?fetchcode' not valid", - "'zzzzz' not valid", - ], + "warnings": [], } ], "packages": [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", "release_date": "2023-12-18", }, { - "purl": "pkg:pypi/fetchcode@0.4.0", + "purl": "pkg:pypi/fetchcode", "release_date": "2024-03-12", "version": "0.4.0", }, { - "purl": "pkg:gem/bundler-sass@0.1.2", + "purl": "pkg:gem/bundler-sass", "version": "0.1.2", "release_date": "2013-12-11", }, @@ -1225,16 +1027,7 @@ def test_versions_cli(self, mock_read_log_file): ), ], ) - @mock.patch("purldb_toolkit.purlcli.read_log_file") - def test_versions_details(self, mock_read_log_file, test_input, expected): - mock_read_log_file.return_value = [ - "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", - "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", - "WARNING - 'pkg:pypi/zzzzz' does not exist in the upstream repo\n", - "WARNING - 'pkg:pypi/?fetchcode' not valid\n", - "WARNING - 'zzzzz' not valid\n", - ] - + def test_versions_details(self, test_input, expected): output = "" file = "" command_name = "versions" @@ -1253,47 +1046,6 @@ def test_versions_details(self, mock_read_log_file, test_input, expected): compare_packages(expected, purl_versions) - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - None, - ), - ( - ["pkg:gem/bundler-sass"], - None, - ), - ( - ["pkg:rubygems/bundler-sass"], - "valid_but_not_supported", - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ( - ["pkg:maven/axis/axis@1.0"], - None, - ), - ], - ) - def test_check_versions_purl(self, test_input, expected): - purl_versions = purlcli.check_versions_purl(test_input[0]) - assert purl_versions == expected - def streamline_metadata_packages(packages): """ From 70e5632e869d9a19a338b5de9cfb8ab83bea15ef Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Sat, 8 Jun 2024 11:19:48 -0700 Subject: [PATCH 7/9] Fix failing tests, clean merged code as needed #65 Reference: Signed-off-by: John M. Horan --- purldb-toolkit/README.rst | 4 ---- purldb-toolkit/src/purldb_toolkit/purlcli.py | 9 ------- purldb-toolkit/tests/test_purlcli_live.py | 25 ++++++++++---------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index f394e585..bbfdd69b 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -154,10 +154,6 @@ This command collects and return a list of all the known versions of a PURL by q and upstream package registries as needed. -This command collects and return a list of all the known versions of a PURL by querying the PurlDB -and upstream package registries as needed. - - Examples ######## diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 2b0cfaaf..2017f63b 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -28,8 +28,6 @@ # in seconds POLLING_INTERVAL = 5 -LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") - @click.group() def purlcli(): @@ -503,13 +501,6 @@ def check_for_duplicate_input_sources(purls, file): raise click.UsageError("Use either purls or file.") -def clear_log_file(): - log_file = Path(LOG_FILE_LOCATION) - - if log_file.is_file(): - os.remove(log_file) - - class D2DPackage(NamedTuple): """ A package to use in d2d, identifier by its PURL and qualified by is package_content which is one diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 53765375..53068965 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -10,7 +10,6 @@ import json import os from collections import OrderedDict -from unittest import mock import pytest from click.testing import CliRunner @@ -162,9 +161,6 @@ def test_metadata_details(self): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -189,6 +185,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -200,9 +199,6 @@ def test_metadata_details(self): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -230,6 +226,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -241,9 +240,6 @@ def test_metadata_details(self): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -271,6 +267,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -282,9 +281,6 @@ def test_metadata_details(self): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -312,6 +308,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), ], From f31dc0e2c6ea8182bf446d1c961ccfa8333072cf Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 13 Jun 2024 10:52:59 -0700 Subject: [PATCH 8/9] Fix test data OrderedDict key order #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/tests/test_purlcli.py | 48 +++++++++++------------ purldb-toolkit/tests/test_purlcli_live.py | 24 ++++++------ 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index a0fc9055..3e8650fb 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -67,6 +67,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -91,9 +94,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -105,6 +105,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -132,9 +135,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -146,6 +146,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -173,9 +176,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -187,6 +187,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -214,9 +217,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), ] @@ -246,6 +246,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -270,9 +273,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -284,6 +284,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -311,9 +314,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -325,6 +325,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -352,9 +355,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -366,6 +366,9 @@ def test_metadata_details(self, mock_collect_metadata): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -393,9 +396,6 @@ def test_metadata_details(self, mock_collect_metadata): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), ], diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 53068965..c1be0410 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -161,6 +161,9 @@ def test_metadata_details(self): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -185,9 +188,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -199,6 +199,9 @@ def test_metadata_details(self): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -226,9 +229,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -240,6 +240,9 @@ def test_metadata_details(self): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -267,9 +270,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -281,6 +281,9 @@ def test_metadata_details(self): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -308,9 +311,6 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), ], From 8f374f50623c7a39dae384ae77b9b627e3f63b76 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 19 Jul 2024 13:09:28 -0700 Subject: [PATCH 9/9] Update metadata-related code to accommodate fetchcode updates #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 12 +- .../purlcli/expected_metadata_output.json | 378 +++--------------- purldb-toolkit/tests/test_purlcli.py | 76 ---- purldb-toolkit/tests/test_purlcli_live.py | 40 +- 4 files changed, 53 insertions(+), 453 deletions(-) diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 2017f63b..b866d934 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -106,12 +106,12 @@ def collect_metadata(purl): from fetchcode/package.py. """ collected_metadata = [] - for release in list(info(purl)): - if release is None: - continue - release_detail = release.to_dict() - release_detail.move_to_end("purl", last=False) - collected_metadata.append(release_detail) + purl_metadata = info(purl) + if purl_metadata: + for release in list(purl_metadata): + release_detail = release.to_dict() + release_detail.move_to_end("purl", last=False) + collected_metadata.append(release_detail) return collected_metadata diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 794022cb..f4300124 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -12,7 +12,7 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:rubygems/rails" + "pkg:rubygems/pronto-goodcheck" ], "--file": null, "--output": "" @@ -22,42 +22,6 @@ } ], "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, { "purl": "pkg:pypi/fetchcode@0.1.0", "type": "pypi", @@ -238,78 +202,6 @@ "repository_download_url": null, "api_data_url": null }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, { "purl": "pkg:pypi/fetchcode@0.3.0", "type": "pypi", @@ -324,80 +216,6 @@ "parties": [], "keywords": [], "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", "download_url": null, "api_url": "https://pypi.org/pypi/fetchcode/json", "size": null, @@ -421,10 +239,10 @@ "api_data_url": null }, { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", + "purl": "pkg:cargo/banquo@0.1.0", + "type": "cargo", "namespace": null, - "name": "fetchcode", + "name": "banquo", "version": "0.1.0", "qualifiers": {}, "subpath": null, @@ -433,20 +251,20 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": null, + "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", + "api_url": "https://crates.io/api/v1/crates/banquo", "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": "BSD-3-Clause", "notice_text": null, "root_path": null, "dependencies": [], @@ -457,57 +275,24 @@ "api_data_url": null }, { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", + "purl": "pkg:rubygems/pronto-goodcheck@0.2.0", + "type": "rubygems", "namespace": null, - "name": "fetchcode", + "name": "pronto-goodcheck", "version": "0.2.0", "qualifiers": {}, "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.2.0.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.2.0.json", "size": null, "sha1": null, "md5": null, @@ -518,32 +303,34 @@ "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": [ + "MIT" + ], "notice_text": null, "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] }, { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", + "purl": "pkg:rubygems/pronto-goodcheck@0.1.2", + "type": "rubygems", "namespace": null, - "name": "fetchcode", - "version": "0.4.0", + "name": "pronto-goodcheck", + "version": "0.1.2", "qualifiers": {}, "subpath": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", + "repository_download_url": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.2.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.2.json", "size": null, "sha1": null, "md5": null, @@ -554,111 +341,41 @@ "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": [ + "MIT" + ], "notice_text": null, "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] }, { - "purl": "pkg:cargo/banquo", - "type": "cargo", + "purl": "pkg:rubygems/pronto-goodcheck@0.1.1", + "type": "rubygems", "namespace": null, - "name": "banquo", - "version": null, + "name": "pronto-goodcheck", + "version": "0.1.1", "qualifiers": {}, "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.1.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.1.json", "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -669,10 +386,7 @@ "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] } ] } diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 3e8650fb..5b898dba 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -58,44 +58,6 @@ def test_metadata_cli_no_input_sources(self): @mock.patch("purldb_toolkit.purlcli.collect_metadata") def test_metadata_details(self, mock_collect_metadata): mock_collect_metadata.return_value = [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"), @@ -237,44 +199,6 @@ def test_metadata_details(self, mock_collect_metadata): } ], "packages": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"), diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index c1be0410..ca14228d 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -50,7 +50,7 @@ def test_metadata_cli(self): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:rubygems/rails", + "pkg:rubygems/pronto-goodcheck", "--output", actual_result_file, ] @@ -152,44 +152,6 @@ def test_metadata_details(self): } ], "packages": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"),