diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b4d70c2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+# File extensions
+*.pyc
+*.pyo
+*~
+*.swp
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..a846902
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,69 @@
+=========
+ COPYING
+=========
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public
+License along with this program, in the file ``licenses/AGPLv3.txt``.
+If not, see .
+
+
+Translation files located under ``mediagoblin/i18n/`` directory tree
+are free software: you can redistribute it and/or modify it under the
+terms of the GNU Affero General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at
+your option) any later version.
+
+You should have received a copy of the GNU Affero General Public
+License along with this program, in the file ``licenses/AGPLv3.txt``.
+If not, see .
+
+
+JavaScript files located in the ``mediagoblin/`` directory tree
+are free software: you can redistribute and/or modify them under the
+terms of the GNU Affero General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at
+your option) any later version.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this program, in the file ``licenses/LGPLv3.txt``.
+If not, see .
+
+
+Documentation files located in the ``docs/`` directory tree and all
+original documentation theme CSS and assets (including image files)
+are released under a CC0 license. To the extent possible under law,
+the author(s) have dedicated all copyright and related and neighboring
+rights to these files to the public domain worldwide. These files are
+distributed without any warranty.
+
+You should have received a copy of the CC0 license in the file
+``licenses/CC0_1.0.txt``. If not, see
+.
+
+
+CSS, images and video located in the ``mediagoblin/`` directory tree are
+released under a CC0 license. To the extent possible under law, the author(s)
+have dedicated all copyright and related and neighboring rights to these
+files to the public domain worldwide. These files are distributed without
+any warranty.
+
+You should have received a copy of the CC0 license in the file
+``licenses/CC0_1.0.txt``. If not, see
+.
+
+
+Additional library software has been made available in the ``extlib/``
+directory. All of it is Free Software and can be distributed under
+liberal terms, but those terms may differ in detail from the AGPL's
+particulars. See each package's license file in the extlib directory
+for additional terms.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..873b20e
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,45 @@
+============================================
+ gmg_localfiles, plugin for GNU MediaGoblin
+============================================
+
+Plugin for importing files from your filesystem without duplication.
+
+This plugin lets you have all your original files in one folder on your file
+system, and it will stop MediaGoblin from copying those files to its own
+locations.
+
+It will try to make mediagoblin not touch/ruin your files (no guarantees!), but
+it will make a `mg_cache` folder in the directory.
+
+Example setup in `mediagoblin.ini`::
+
+ [storage:queuestore]
+ base_dir = /srv/media/Pictures
+ storage_class = gmg_localfiles.storage:PersistentFileStorage
+
+ [storage:publicstore]
+ base_dir = /srv/media/Pictures
+ base_url = /mgoblin_media/
+ storage_class = gmg_localfiles.storage:PersistentFileStorage
+
+ [plugins]
+ [[gmg_localfiles]]
+
+You will also need to serve the files, so in `paste.ini`::
+
+ [app:publicstore_serve]
+ use = egg:Paste#static
+ document_root = %(here)s/user_dev/media/public/
+
+--------------
+ Installation
+--------------
+
+Put gmg_localfiles somewhere on your Python path. You might even just put it
+inside the MediaGoblin folder if you want to be done with it quickly ;-)
+
+---------
+ Running
+---------
+
+Go into the `gmg_localfiles` folder and run `python import_files.py`.
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..ee70eda
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+#
+# GMG localfiles plugin -- local file import
+# Copyright (C) 2012 Odin Hørthe Omdal
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import os
+import sys
+import logging
+
+from mediagoblin.tools.pluginapi import get_config
+from mediagoblin import processing
+
+_log = logging.getLogger(__name__)
+
+
+# Monkeypatch create_pub_filepath to not clean the original files, and to
+# rather use queued_media_file instead of hardcoded path.
+from storage import _is_cachefile
+from mediagoblin.storage import clean_listy_filepath
+def monkey_create_pub_filepath(entry, filename):
+ if _is_cachefile(filename):
+ filepath = clean_listy_filepath(entry.queued_media_file)
+ else:
+ filepath = list(entry.queued_media_file)
+
+ filepath[-1] = filename
+ return filepath
+processing.create_pub_filepath = monkey_create_pub_filepath
+
+
+class PreservingFilenameBuilder(processing.FilenameBuilder):
+ def __init__(self, path):
+ """Initialize a builder from an original file path."""
+ self.dirpath, self.basename = os.path.split(path)
+ self.basename, self.ext = os.path.splitext(self.basename)
+
+ def fill(self, fmtstr):
+ basename_len = (self.MAX_FILENAME_LENGTH -
+ len(fmtstr.format(basename='', ext=self.ext)))
+ ext = self.ext
+ if _is_cachefile(fmtstr):
+ ext = ext.lower()
+ return fmtstr.format(basename=self.basename[:basename_len],
+ ext=ext)
+processing.FilenameBuilder = PreservingFilenameBuilder
+
+
+def setup_plugin():
+ _log.info('LocalFiles plugin set up!')
+ config = get_config('gmg_localfiles')
+ if not config:
+ _log.info('There is no configuration set.')
+ sys.exit(1)
+
+hooks = {
+ 'setup': setup_plugin
+ }
diff --git a/import_files.py b/import_files.py
new file mode 100644
index 0000000..a7da57d
--- /dev/null
+++ b/import_files.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GMG localfiles plugin -- local file import
+# Copyright (C) 2012 Odin Hørthe Omdal
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+
+# This is here early because of a race
+from mediagoblin.app import MediaGoblinApp
+if __name__ == "__main__":
+ config_file = '/home/odin/src/mediagoblin/mediagoblin.ini'
+ mg = MediaGoblinApp(config_file, setup_celery=True)
+ from mediagoblin import mg_globals
+
+ from mediagoblin.init.celery import setup_celery_app
+ setup_celery_app(mg_globals.app_config, \
+ mg_globals.global_config, force_celery_always_eager=True)
+
+import os
+import uuid
+
+from celery import registry
+
+from mediagoblin.tools.text import convert_to_tag_list_of_dicts
+from mediagoblin.storage import clean_listy_filepath
+from mediagoblin.processing import mark_entry_failed
+from mediagoblin.processing.task import ProcessMedia
+from mediagoblin.media_types import sniff_media, \
+ InvalidFileType, FileTypeNotSupported
+
+
+class MockMedia():
+ filename = ""
+ stream = None
+ def __init__(self, filename, stream):
+ self.filename = filename
+ self.stream = stream
+
+
+class ImportCommand(object):
+ #args = ''
+ help = 'Find new photos and add to database'
+
+ def __init__(self, db, base_dir, **kwargs):
+ self.db = db
+ self.base_dir = base_dir
+
+ def handle(self):
+ #Photo.objects.all().delete()
+
+ os.chdir(self.base_dir)
+
+ for top, dirs, files in os.walk(u'.'):
+ # Skip hidden folders
+ if '/.' in top:
+ continue
+ # Skip cache folders
+ if '_cache' in top:
+ print "cache skip", top
+ continue
+ if top == ".":
+ top = ""
+
+ #folder, new_folder = Folder.objects.select_related("photos") \
+ # .get_or_create(path=os.path.normpath(top) + "/",
+ # defaults={'name': os.path.basename(top)})
+ folder_path = os.path.normpath(top)
+ try:
+ cleaned_top = "/".join(clean_listy_filepath(folder_path.split("/")))
+ except Exception:
+ cleaned_top = top
+ new_folder = not os.path.exists(os.path.join("mg_cache", cleaned_top))
+
+ if not new_folder:
+ print u"Skipping folder {0}".format(folder_path).encode("utf-8")
+ continue
+ new_files = [os.path.splitext(i)[0] for i in files]
+ new_files.sort(reverse=True)
+
+ for new_filename in new_files:
+ file_url = os.path.join(folder_path, new_filename)
+
+ # More than one file with the same name but different
+ # extension?
+ exts = [os.path.splitext(f)[1] for f in files if new_filename
+ in f]
+
+ assert len(exts) > 0, "Couldn't find file extension for %s" % file_url
+
+ # If there exists NEF file, prefer that as canonical file
+ if '.nef' in exts and os.path.exists(file_url + '.nef'):
+ f = file_url + '.nef'
+ elif '.NEF' in exts and os.path.exists(file_url + '.NEF'):
+ f = file_url + '.NEF'
+ else:
+ f = file_url + exts[0]
+
+ try:
+ m = MockMedia(filename=f, stream=open(f, "r"))
+ self.import_file(m)
+ except Exception as e:
+ print u"file: {0} exception: {1}".format(f, e).encode('utf-8')
+ continue
+
+
+ def import_file(self, media):
+ try:
+ media_type, media_manager = sniff_media(media)
+ except (InvalidFileType, FileTypeNotSupported) as e:
+ print u"File error {0}: {1}".format(media.filename, repr(e)).encode("utf-8")
+ return
+ entry = self.db.MediaEntry()
+ entry.media_type = unicode(media_type)
+ entry.title = unicode(os.path.splitext(media.filename)[0])
+
+ entry.uploader = 1
+ # Process the user's folksonomy "tags"
+ entry.tags = convert_to_tag_list_of_dicts("")
+ # Generate a slug from the title
+ entry.generate_slug()
+
+ task_id = unicode(uuid.uuid4())
+
+ entry.queued_media_file = media.filename.split("/")
+ entry.queued_task_id = task_id
+
+ entry.save(validate=True)
+
+ process_media = registry.tasks[ProcessMedia.name]
+ try:
+ process_media.apply_async( [unicode(entry.id)], {}, task_id=task_id)
+ except BaseException as exc:
+ mark_entry_failed(entry.id, exc)
+ raise
+
+
+if __name__ == "__main__":
+ from mediagoblin import mg_globals
+ ic = ImportCommand(mg.db, mg_globals.global_config['storage:publicstore']['base_dir'])
+ ic.handle()
diff --git a/storage.py b/storage.py
new file mode 100644
index 0000000..1bbd5f4
--- /dev/null
+++ b/storage.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# GMG localfiles plugin -- local file import
+# Copyright (C) 2012 Odin Hørthe Omdal
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+from mediagoblin.storage import (
+ clean_listy_filepath,
+ NoWebServing)
+from mediagoblin.storage.filestorage import BasicFileStorage
+
+import os
+import shutil
+import logging
+import urlparse
+
+_log = logging.getLogger(__name__)
+
+def _is_cachefile(filepath):
+ if not isinstance(filepath, basestring):
+ filepath = filepath[-1]
+ return any(k in filepath for k in ['thumbnail', 'medium'])
+
+class PersistentFileStorage(BasicFileStorage):
+ """
+ Local filesystem implementation of storage API that doesn't delete files
+ """
+
+ def _resolve_filepath(self, filepath, force_cache=False):
+ """
+ Transform the given filepath into a local filesystem filepath.
+ """
+ if _is_cachefile(filepath) or force_cache:
+ filepath = clean_listy_filepath(list(filepath))
+ filepath.insert(0, "mg_cache")
+
+ return os.path.join(
+ self.base_dir, *filepath)
+
+ def file_url(self, filepath):
+ if not self.base_url:
+ raise NoWebServing(
+ "base_url not set, cannot provide file urls")
+
+ if _is_cachefile(filepath):
+ filepath = clean_listy_filepath(list(filepath))
+ filepath.insert(0, "mg_cache")
+
+ return urlparse.urljoin(
+ self.base_url,
+ '/'.join(filepath))
+
+ def get_file(self, filepath, mode='r'):
+ if _is_cachefile(filepath):
+ return super(PersistentFileStorage, self).get_file(filepath, mode)
+ if not os.path.exists(self._resolve_filepath(filepath)):
+ return PersistentStorageObjectWrapper(None, self._resolve_filepath(filepath))
+
+ mode = mode.replace("w", "r")
+ # Grab and return the file in the mode specified
+ return PersistentStorageObjectWrapper(
+ open(self._resolve_filepath(filepath), mode))
+
+ def delete_file(self, filepath):
+ #os.remove(self._resolve_filepath(filepath))
+ _log.info(u'Not removing {0} as requested.'.format(self._resolve_filepath(filepath)))
+
+ def copy_local_to_storage(self, filename, filepath):
+ """
+ Copy this file from locally to the storage system.
+ """
+ # Make directories if necessary
+ if len(filepath) > 1:
+ directory = self._resolve_filepath(filepath[:-1], force_cache=True)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ shutil.copy(filename, self.get_local_path(filepath))
+
+class PersistentStorageObjectWrapper():
+ def __init__(self, storage_object, name=None, *args, **kwargs):
+ self.storage_object = storage_object
+ self.name = name
+ if storage_object:
+ self.name = storage_object.name
+
+ def read(self, *args, **kwargs):
+ _log.debug(u'Reading {0}'.format(
+ self.name).encode("utf-8"))
+ return self.storage_object.read(*args, **kwargs)
+
+ def write(self, data, *args, **kwargs):
+ _log.debug(u'Not writing {0}'.format(
+ self.name).encode("utf-8"))
+
+ def close(self):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc_info):
+ self.close()