diff --git a/CHANGELOG.md b/CHANGELOG.md index 70af6c1..51f35d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Current (in progress) -- Nothing yet +- Enable creating a static resource from a URL [#26](https://github.com/datagouv/csv-detective/pull/26) ## 0.1.4 (2025-09-04) diff --git a/README.md b/README.md index d83d88e..a5e5f9e 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,18 @@ resource = dataset.create_remote( # to update the file of a static resource resource.update({"title": "New title"}, file_to_upload="path/to/your/new_file.txt") ``` + +You can also create a static resource from a file using its URL by setting `from_url=True`, in which case the `file_name` and `mime` arguments are required: +```python +resource = dataset.create_static( + file_to_upload="https://wesite.org/path/to/your/file.txt", + payload={"title": "New static resource"}, + from_url=True, + file_name="file.txt", + mime="text/plain", +) # this creates a static resource with the values you specified, and instantiates a Resource +``` + > **Note:** If you are not planning to use an object's attributes, you may prevent the initial API call using `fetch=False`, in order not to unnecessarily ping the API. ```python dataset = client.dataset("5d13a8b6634f41070a43dff3", fetch=False) diff --git a/datagouv/resource.py b/datagouv/resource.py index 84393b3..3336e9d 100755 --- a/datagouv/resource.py +++ b/datagouv/resource.py @@ -151,10 +151,13 @@ def create_remote( @simple_connection_retry def create_static( self, - file_to_upload: str, # the path of the file + file_to_upload: str, # the path or URL of the file payload: dict, dataset_id: str | None = None, is_communautary: bool = False, + from_url: bool = False, + file_name: str | None = None, + mime: str | None = None, ) -> Resource: if dataset_id and self.__class__.__name__ == "Dataset": raise ValueError( @@ -170,9 +173,26 @@ def create_static( if is_communautary: url += "community/" logging.info(f"🆕 Creating '{payload['title']}' for {file_to_upload}") - r = self._client.session.post(url, files={"file": open(file_to_upload, "rb")}) - r.raise_for_status() - metadata = r.json() + if from_url: + if file_name is None or mime is None: + raise ValueError( + "When uploading from a URL, file_name and mime arguments are required." + ) + with self._client.session.stream("GET", file_to_upload) as streamed: + streamed.raise_for_status() + streamed.read() + with self._client.session.stream( + "POST", + url, + files={"file": (file_name, streamed.content, mime)}, + ) as r: + r.raise_for_status() + r.read() + metadata = r.json() + else: + r = self._client.session.post(url, files={"file": open(file_to_upload, "rb")}) + r.raise_for_status() + metadata = r.json() resource_id = metadata["id"] r = Resource( id=resource_id, diff --git a/tests/test_resource.py b/tests/test_resource.py index 00bdeba..993d8b2 100755 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -54,7 +54,7 @@ def test_resource_attributes_and_methods(static_resource_api2_call): def test_authentification_assertion(): client = Client() with pytest.raises(PermissionError): - client.resource().create_static({"path": "path"}, {"title": "Titre"}, DATASET_ID) + client.resource().create_static("path/to/file.csv", {"title": "Titre"}, DATASET_ID) with pytest.raises(PermissionError): client.resource().create_remote({"url": "url", "title": "Titre"}, DATASET_ID) r_from_response = Resource( @@ -111,3 +111,14 @@ def test_resource_download(remote_resource_api1_call, file_name, httpx_mock): assert rows[0] == "a,b,c\n" assert rows[1] == "1,2,3" os.remove(local_name) + + +def test_create_static_from_url(): + client = Client(api_key="SUPER_SECRET") + with pytest.raises(ValueError): + client.resource().create_static( + "https://example.com/file.csv", + {"title": "Titre"}, + DATASET_ID, + from_url=True, + )