Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

patch spot67 #519

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

patch spot67 #519

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3756d4
patch spot67
boney-bun Mar 21, 2018
9c3b1b2
Merge branch 'master' into patch-spot67
boney-bun Jul 31, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 157 additions & 135 deletions django_project/catalogue/ingestors/spot67.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
__author__ = 'rischan - <[email protected]>'
__date__ = '4/28/16'


import os
import sys
import glob
Expand Down Expand Up @@ -233,6 +232,7 @@ def get_radiometric_resolution():
"""Get the radiometric resolution for the supplied product record."""
return 12 # static value based on client information


def get_projection():
# If projection not found default to WGS84
projection = Projection.objects.get(epsg_code=4326)
Expand Down Expand Up @@ -312,126 +312,130 @@ def log_message(message, level=1):
log_message('Starting directory scan...', 2)

for myFolder in glob.glob(os.path.join(source_path, '*')):
record_count += 1
try:
log_message('', 2)
log_message('---------------', 2)
log_message('', 1)
log_message('---------------', 1)
# Get the folder name
product_folder = os.path.split(myFolder)[-1]
log_message(product_folder, 2)
log_message(product_folder, 1)

# Find the first and only xml file in the folder
search_path = os.path.join(str(myFolder), '*.xml')
log_message(search_path, 2)
xml_file = glob.glob(search_path)[0]
log_message(xml_file, 2)

# Create a DOM document from the file
dom = parse(xml_file)
#
# First grab all the generic properties that any spot will have...
geometry = get_geometry(log_message, dom)
start_date_time, center_date_time = get_dates(
log_message, dom)
# projection for GenericProduct
projection = get_projection()
original_product_id = get_original_product_id(dom)
# Band count for GenericImageryProduct
band_count = get_band_count()
orbit_number = get_orbit_number(dom)
# # Spatial resolution x for GenericImageryProduct
spatial_resolution_x = float(get_spatial_resolution_x())
# # Spatial resolution y for GenericImageryProduct
spatial_resolution_y = float(
get_spatial_resolution_y())
log_message('Spatial resolution y: %s' % spatial_resolution_y, 2)
#
# # Spatial resolution for GenericImageryProduct calculated as (x+y)/2
spatial_resolution = (spatial_resolution_x + spatial_resolution_y) / 2
log_message('Spatial resolution: %s' % spatial_resolution, 2)
radiometric_resolution = get_radiometric_resolution()
log_message('Radiometric resolution: %s' % radiometric_resolution, 2)
quality = get_quality()
# ProductProfile for OpticalProduct
product_profile = get_product_profile(log_message, dom)
# Get the original text file metadata
metadata_file = file(xml_file, 'rt')
metadata = metadata_file.readlines()
metadata_file.close()
log_message('Metadata retrieved', 2)

unique_product_id = original_product_id
# Check if there is already a matching product based
# on original_product_id

# Do the ingestion here...
data = {
'metadata': metadata,
'spatial_coverage': geometry,
'radiometric_resolution': radiometric_resolution,
'band_count': band_count,
'original_product_id': original_product_id,
'unique_product_id': unique_product_id,
'spatial_resolution_x': spatial_resolution_x,
'spatial_resolution_y': spatial_resolution_y,
'spatial_resolution': spatial_resolution,
'product_profile': product_profile,
'product_acquisition_start': start_date_time,
'product_date': center_date_time,
'orbit_number': orbit_number,
'projection': projection,
'quality': quality
}
log_message(data, 3)
# Check if it's already in catalogue:
try:
today = datetime.today()
time_stamp = today.strftime("%Y-%m-%d")
log_message('Time Stamp: %s' % time_stamp, 2)
except Exception, e:
print e.message

update_mode = True
try:
log_message('Trying to update')
#original_product_id is not necessarily unique
#so we use product_id
product = OpticalProduct.objects.get(
original_product_id=original_product_id
).getConcreteInstance()
log_message(('Already in catalogue: updating %s.'
% original_product_id), 2)
new_record_flag = False
message = product.ingestion_log
message += '\n'
message += '%s : %s - updating record' % (
time_stamp, ingestor_version)
data['ingestion_log'] = message
product.__dict__.update(data)
except ObjectDoesNotExist:
log_message('Not in catalogue: creating.', 2)
update_mode = False
message = '%s : %s - creating record' % (
time_stamp, ingestor_version)
data['ingestion_log'] = message
log_message('search path: %s' % search_path, 1)
# xml_file = glob.glob(search_path)[0]
for xml_file in glob.glob(search_path):
record_count += 1
log_message(xml_file, 1)
# Create a DOM document from the file
file_path = os.path.basename(xml_file)
filename = os.path.splitext(file_path)[0]

dom = parse(xml_file)
#
# First grab all the generic properties that any spot will have...
geometry = get_geometry(log_message, dom)
start_date_time, center_date_time = get_dates(
log_message, dom)
# projection for GenericProduct
projection = get_projection()
original_product_id = get_original_product_id(dom)
# Band count for GenericImageryProduct
band_count = get_band_count()
orbit_number = get_orbit_number(dom)

# # Spatial resolution x for GenericImageryProduct
spatial_resolution_x = float(get_spatial_resolution_x())
# # Spatial resolution y for GenericImageryProduct
spatial_resolution_y = float(
get_spatial_resolution_y())
log_message('Spatial resolution y: %s' % spatial_resolution_y, 2)
#
# # Spatial resolution for GenericImageryProduct calculated as (x+y)/2
spatial_resolution = (spatial_resolution_x + spatial_resolution_y) / 2
log_message('Spatial resolution: %s' % spatial_resolution, 2)
radiometric_resolution = get_radiometric_resolution()
log_message('Radiometric resolution: %s' % radiometric_resolution, 2)
quality = get_quality()
# ProductProfile for OpticalProduct
product_profile = get_product_profile(log_message, dom)
# Get the original text file metadata
metadata_file = file(xml_file, 'rt')
metadata = metadata_file.readlines()
metadata_file.close()
log_message('Metadata retrieved', 2)

unique_product_id = original_product_id
# Check if there is already a matching product based
# on original_product_id

# Do the ingestion here...
data = {
'metadata': metadata,
'spatial_coverage': geometry,
'radiometric_resolution': radiometric_resolution,
'band_count': band_count,
'original_product_id': original_product_id,
'unique_product_id': unique_product_id,
'spatial_resolution_x': spatial_resolution_x,
'spatial_resolution_y': spatial_resolution_y,
'spatial_resolution': spatial_resolution,
'product_profile': product_profile,
'product_acquisition_start': start_date_time,
'product_date': center_date_time,
'orbit_number': orbit_number,
'projection': projection,
'quality': quality
}
log_message(data, 3)
# Check if it's already in catalogue:
try:
product = OpticalProduct(**data)
log_message('Product: %s' % product)

today = datetime.today()
time_stamp = today.strftime("%Y-%m-%d")
log_message('Time Stamp: %s' % time_stamp, 2)
except Exception, e:
log_message(e.message, 2)
print e.message

new_record_flag = True
except Exception, e:
print e.message
update_mode = True
try:
log_message('Trying to update')
#original_product_id is not necessarily unique
#so we use product_id
product = OpticalProduct.objects.get(
original_product_id=original_product_id
).getConcreteInstance()
log_message(('Already in catalogue: updating %s.'
% original_product_id), 1)
new_record_flag = False
message = product.ingestion_log
message += '\n'
message += '%s : %s - updating record' % (
time_stamp, ingestor_version)
data['ingestion_log'] = message
product.__dict__.update(data)
except ObjectDoesNotExist:
log_message('Not in catalogue: creating.', 1)
update_mode = False
message = '%s : %s - creating record' % (
time_stamp, ingestor_version)
data['ingestion_log'] = message
try:
product = OpticalProduct(**data)
log_message('Product: %s' % product)

except Exception, e:
log_message(e.message, 2)

new_record_flag = True
except Exception, e:
print e.message

log_message('Saving product and setting thumb', 2)
try:
product.save()
if update_mode:
updated_record_count += 1
else:
created_record_count += 1
log_message('Saving product and setting thumb', 1)
try:
product.save()
if update_mode:
updated_record_count += 1
else:
created_record_count += 1

if test_only_flag:
log_message('Testing: image not saved.', 2)
Expand All @@ -449,37 +453,55 @@ def log_message(message, level=1):
pass

jpeg_path = os.path.join(str(myFolder))
jpeg_path = jpeg_path.replace(".XML", "-THUMB.JPG")
log_message(jpeg_path, 1)
jpeg_path = jpeg_path.replace(".XML", "-THUMB.jpg")

if jpeg_path:
new_name = '%s.JPG' % product.original_product_id
shutil.copyfile(
os.path.join(jpeg_path, new_name),
os.path.join(thumbs_folder,new_name))
print new_name
try:
new_name = '%s.jpg' % product.original_product_id
log_message(new_name, 1)
shutil.copyfile(
os.path.join(jpeg_path, new_name),
os.path.join(thumbs_folder, new_name))
except IOError as e:
if ignore_missing_thumbs:
failed_record_count += 1
continue
else:
raise Exception('Missing thumbnail in %s' % jpeg_path)

# Transform and store .wld file
# log_message('Referencing thumb', 2)
# try:
# path = product.georeferencedThumbnail()
# log_message('Georeferenced Thumb: %s' % path, 2)
# except:
# traceback.print_exc(file=sys.stdout)
# elif ignore_missing_thumbs:
# log_message('IGNORING missing thumb:')
else:
raise Exception('Missing thumbnail in %s' % jpeg_path)

if new_record_flag:
log_message('Product %s imported.' % record_count, 2)
pass
if new_record_flag:
log_message('Product %s imported.' % record_count, 2)
pass
else:
log_message('Product %s updated.' % updated_record_count, 2)
pass
except Exception, e:
traceback.print_exc(file=sys.stdout)
raise CommandError('Cannot import: %s' % e)

if test_only_flag:
transaction.rollback()
log_message('Imported scene : %s' % xml_file, 1)
log_message('Testing only: transaction rollback.', 1)
else:
log_message('Product %s updated.' % updated_record_count, 2)
pass
except Exception, e:
traceback.print_exc(file=sys.stdout)
raise CommandError('Cannot import: %s' % e)

if test_only_flag:
transaction.rollback()
log_message('Imported scene : %s' % product_folder, 1)
log_message('Testing only: transaction rollback.', 1)
else:
transaction.commit()
log_message('Imported scene : %s' % product_folder, 1)
except Exception, e:
transaction.commit()
log_message('Imported scene : %s' % xml_file, 1)
except Exception as e:
log_message('Record import failed. AAAAAAARGH! : %s' %
product_folder, 1)
e, 1)
failed_record_count += 1
if halt_on_error_flag:
print e.message
Expand Down