Create a preview for a text file with the first 1MB text in the file.

text/preview/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.8
+
+WORKDIR /extractor
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
+
+COPY text_preview_extractor.py extractor_info.json ./
+CMD python text_preview_extractor.py

text/preview/extractor_info.json

@@ -0,0 +1,24 @@
+{
+  "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld",
+  "name": "ncsa.text.preview",
+  "version": "1.0",
+  "description": "Given a text file, keep the first 1MB of text.",
+  "author": "Luigi Marini <[email protected]>",
+  "contributors": [],
+  "contexts": [
+  ],
+  "repository": [
+    {
+      "repType": "git",
+      "repUrl": "https://github.com/clowder-framework/extractors-core.git"
+    }
+  ],
+  "process": {
+    "file": [
+      "text/*"
+    ]
+  },
+  "external_services": [],
+  "dependencies": [],
+  "bibtex": []
+}

text/preview/requirements.txt

@@ -0,0 +1 @@
+pyclowder==2.7.0

text/preview/simple/Dockerfile

@@ -0,0 +1,5 @@
+ARG PYCLOWDER_PYTHON=""
+FROM clowder/extractors-simple-extractor${PYCLOWDER_PYTHON}:onbuild
+
+ENV EXTRACTION_FUNC="simpletextpreview"
+ENV EXTRACTION_MODULE="simpletextpreview"

text/preview/simple/extractor_info.json

@@ -0,0 +1,24 @@
+{
+  "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld",
+  "name": "ncsa.text.preview",
+  "version": "1.0",
+  "description": "Given a text file, keep the first 1MB of text.",
+  "author": "Luigi Marini <[email protected]>",
+  "contributors": [],
+  "contexts": [
+  ],
+  "repository": [
+    {
+      "repType": "git",
+      "repUrl": "https://github.com/clowder-framework/extractors-core.git"
+    }
+  ],
+  "process": {
+    "file": [
+      "text/*"
+    ]
+  },
+  "external_services": [],
+  "dependencies": [],
+  "bibtex": []
+}

text/preview/simple/simpletextpreview.py

@@ -0,0 +1,37 @@
+"""This version does not currently work. Just leaving it here until I can figure out why not."""
+
+import os
+import tempfile
+
+
+def textpreview(input_file_path):
+    """
+    This function keeps the first 1000 lines of a text files and uploads them as a preview to the file.
+
+    :param input_file_path: Full path to the input file
+    :return: Result dictionary containing the path on disk of the preview
+    """
+    numlines = 10
+
+    with open(input_file_path, 'r') as file:
+        lines = file.readlines(numlines)
+        with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
+            for line in lines:
+                print("Line " + line)
+                tmp.write(line)
+
+        # (fd, tmp) = tempfile.mkstemp(suffix='.txt')
+        # with open(tmp, 'w') as f:
+        #     for line in lines:
+        #         print("Line " + line)
+        #         f.write(line)
+        # os.close(fd)
+
+    # return the path of preview on disk
+    print("Preview path " + tmp.name)
+    result = {
+        'previews': [tmp.name]
+    }
+
+    # Return the result dictionary
+    return result

text/preview/text_preview_extractor.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+import logging
+import tempfile
+
+import pyclowder
+from pyclowder.extractors import Extractor
+
+
+class TextPreviewExtractor(Extractor):
+
+    def __init__(self):
+        Extractor.__init__(self)
+        self.setup()
+
+        # setup logging for the extractor
+        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
+        logging.getLogger('__main__').setLevel(logging.DEBUG)
+
+    def process_message(self, connector, host, secret_key, resource, parameters):
+        # Process the file and upload the results
+
+        inputfile = resource["local_paths"][0]
+        file_id = resource['id']
+
+        # 1 MB
+        num_bytes = 1000000
+
+        with open(inputfile, 'r') as file:
+            lines = file.readlines(num_bytes)
+            tmp = tempfile.NamedTemporaryFile(suffix=".txt", mode="w", delete=False, dir='./')
+            tmp.write("Previewing up to the first 1 megabyte of file contents. Download file to see full data.\n---\n")
+            for line in lines:
+                tmp.write(line)
+            tmp.close()
+
+        pyclowder.files.upload_preview(connector, host, secret_key, file_id, tmp.name, None)
+        connector.status_update(pyclowder.utils.StatusMessage.processing, resource,
+                                "Uploaded preview of type txt")
+
+
+if __name__ == "__main__":
+    extractor = TextPreviewExtractor()
+    extractor.start()