Checks for erroneous data; new data types; reduced the number of photos needed to trigger a shorter timespan

flickrhistory/databaseobjects.py

@@ -170,6 +170,11 @@ class FlickrPhoto(Base):
     user_id = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False)
     user_farm = sqlalchemy.Column(sqlalchemy.SmallInteger, nullable=False)
 
+    # New fields for tags and license
+    tags = sqlalchemy.Column(sqlalchemy.Text)
+    license = sqlalchemy.Column(sqlalchemy.Integer)
+    geo_accuracy = sqlalchemy.Column(sqlalchemy.Integer)
+
     user = sqlalchemy.orm.relationship("FlickrUser", back_populates="photos")
 
     __table_args__ = (
@@ -181,6 +186,15 @@ class FlickrPhoto(Base):
     @classmethod
     def from_raw_api_data_flickrphotossearch(cls, data):
         """Initialise a new FlickrPhoto with a flickr.photos.search data dict."""
+
+        # Helper function to clean NUL characters
+        def clean_string(input_string):
+            return (
+                input_string.replace("\x00", "")
+                if isinstance(input_string, str)
+                else input_string
+            )
+
         # the API does not always return all fields
         # we need to figure out which ones we can use
 
@@ -209,12 +223,12 @@ def from_raw_api_data_flickrphotossearch(cls, data):
             pass
 
         try:
-            photo_data["title"] = data["title"]
+            photo_data["title"] = clean_string(data["title"])
         except KeyError:
             pass
 
         try:
-            photo_data["description"] = data["description"]["_content"]
+            photo_data["description"] = clean_string(data["description"]["_content"])
         except KeyError:
             pass
 
@@ -254,6 +268,21 @@ def from_raw_api_data_flickrphotossearch(cls, data):
         ):
             pass
 
+        try:
+            photo_data["tags"] = clean_string(data["tags"])
+        except KeyError:
+            pass
+
+        try:
+            photo_data["license"] = int(data["license"])
+        except (ValueError, KeyError):
+            pass
+
+        try:
+            photo_data["geo_accuracy"] = int(data["accuracy"])
+        except (ValueError, KeyError):
+            pass
+
         # finally, the user
         # (let’s just delegate that to the FlickrUser constructor)
         photo_data["user"] = FlickrUser.from_raw_api_data_flickrphotossearch(data)

flickrhistory/photodownloader.py

@@ -52,7 +52,15 @@ def photos(self):
             "per_page": 500,
             "has_geo": 1,
             "extras": ", ".join(
-                ["description", "date_upload", "date_taken", "geo", "owner_name"]
+                [
+                    "description",
+                    "date_upload",
+                    "date_taken",
+                    "geo",
+                    "owner_name",
+                    "tags",
+                    "license",
+                ]
             ),
             "min_upload_date": self._timespan.start.timestamp(),
             "max_upload_date": self._timespan.end.timestamp(),
@@ -81,17 +89,21 @@ def photos(self):
                     # unsuccessful and start over
                     raise ApiResponseError() from exception
 
+            # Check for 'photos' in results to avoid KeyError
+            if "photos" not in results or "photo" not in results["photos"]:
+                break
+
             try:
                 num_photos = int(results["photos"]["total"])
             except TypeError:
                 num_photos = 0
 
-            if num_photos > 4000 and self._timespan.duration > datetime.timedelta(
+            if num_photos > 3000 and self._timespan.duration > datetime.timedelta(
                 seconds=1
             ):
                 raise DownloadBatchIsTooLargeError(
                     (
-                        "More than 4000 rows returned ({:d}), "
+                        "More than 3000 rows returned ({:d}), "
                         + "please specify a shorter time span."
                     ).format(num_photos)
                 )

setup.cfg

@@ -23,7 +23,7 @@ install_requires =
     blessed
     GeoAlchemy2
     PyYAML
-    psycopg2
+    psycopg2-binary
     requests
     SQLAlchemy >= 1.4.0b1
     urllib3