more on uploading files / created a new collection to split columns f…

…rom dsi

README.md

@@ -4,7 +4,7 @@
 -------
 ## PRESENTATION
 
-part 2a/3 of the TADATA! sofware suite (ApiViz / Solidata / OpenScraper)
+part 2a/3 of the TADATA! sofware suite ([ApiViz](https://github.com/entrepreneur-interet-general/CIS-front) / Solidata / [OpenScraper](https://github.com/entrepreneur-interet-general/OpenScraper) )
 
 #### Building a public service for managing, consolidating, and sharing data 
 
@@ -17,7 +17,7 @@ part 2a/3 of the TADATA! sofware suite (ApiViz / Solidata / OpenScraper)
 - consolidate your data : apply your own datamodel, simplify columns, apply metadatas, ... ; 
 - share the consolidated datas on API endpoints with the level of openness you decide (opendata, commons, collective, private) ;
 - manage all your data and your recipes by projects ;
-- be able to manage projects by teams and share data/recipes/
+- be able to manage projects by teams and share data/recipes/datamodels...
 
 
 ------
@@ -95,5 +95,12 @@ part 2a/3 of the TADATA! sofware suite (ApiViz / Solidata / OpenScraper)
 ![alt text](./screenshots/endpoints_users.png "endpoint users")
 
 -------
-![alt text](./screenshots/endpoints_auth_server.png "endpoint users")
+![alt text](./screenshots/endpoints_auth_server.png "endpoint auth users")
+
+-------
+![alt text](./screenshots/endpoints_projects.png "endpoint projects")
+
+-------
+![alt text](./screenshots/endpoints_dataset_inputs.png "endpoint dataset inputs")
+
 

solidata_api/_choices/__init__.py

@@ -1,4 +1,6 @@
 
 from ._choices_files import *
+from ._choices_open_level import *
 from ._choices_licences import *
-from ._choices_user import *
+from ._choices_user import *
+from._choices_docs import *

solidata_api/_choices/_choices_docs.py

@@ -0,0 +1,18 @@
+
+# -*- encoding: utf-8 -*-
+
+"""
+_choices_docs.py  
+- all choices related to documents
+"""
+# from copy import copy, deepcopy
+
+from log_config import log, pformat
+
+log.debug("... loading _choices_docs.py ...")
+
+
+
+doc_src_type_list = ["api","xls","xlsx","xml","csv"]
+
+doc_type_list			= ["usr","prj","dmt","dmf","dsi","dsr","rec","dso","tag"]

solidata_api/_choices/_choices_files.py

@@ -11,6 +11,7 @@
 log.debug("... loading _choices_files.py ...")
 
 
+
 authorized_filetypes = [
 	 "csv", "xls", "xlsx", "xml" # ...
 ]

solidata_api/_choices/_choices_open_level.py

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+
+"""
+_choices_open_level.py  
+- all choices related to open data levels
+"""
+# from copy import copy, deepcopy
+
+from log_config import log, pformat
+
+log.debug("... loading _choices_open_level.py ...")
+
+
+open_level_choices = [
+  "open_data",
+  "collective",
+  "commons",
+  "private"
+]

solidata_api/_choices/_choices_user.py

@@ -11,6 +11,8 @@
 log.debug("... loading _choices_user.py ...")
 
 
+from ._choices_docs import * 
+
 ### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
 ### CHOICES ONLY FOR ADMIN
 ### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
@@ -20,13 +22,13 @@
 	"staff",   ### can edit all datamodels : dmf + dmt
 	# "collective", 
 	"registred", 
-	"guest",
+	"guest", ### not registred yet
 	"anonymous"
 ]
 
 bad_passwords = [ 
 	'test', 
-	'password', 
+	'password', 'Password',
 	'12345' 
 ]
 
@@ -35,7 +37,6 @@
 	"can_edit_dmt",
 	"can_edit_dmf",
 	"can_edit_dsi",
-	# "can_edit_dc",
 	"can_edit_rec",
 	"can_only_view"
 ]
@@ -49,12 +50,13 @@
 ### user fields as recorded in DB - most exhaustive
 ### check 'schema_users.py' for coherence and description
 user_fields_admin_can_update = {
-	"infos" 				: ["name", "surname", "email"], 
-	"auth" 					: ["pwd", "conf_usr", "role", "refr_tok", "blklst_usr"],
-	"preferences" 	: ["lang", "fav_list"],
-	"datasets" 			: ["proj_", "dm_", "dsi_", "dso_", "dc_", "rec_"],
-	"profile" 			: ["profiles"],
-	"professional" 	: ["struct", "struct_profiles"]
+	"infos" 							: ["name", "surname", "email"], 
+	"auth" 								: ["pwd", "conf_usr", "role", "refr_tok", "is_blacklisted"],
+	# "preferences" 		: ["lang", "fav_list"],
+	# "datasets" 				: doc_type_list,
+	"datasets"						: [ ds+"_list" for ds in doc_type_list ],
+	"profile" 						: ["lang", "usr_profiles"],
+	"professional_infos"	: ["structure", "struct_profiles", "structure_url"]
 }
 
 
@@ -87,6 +89,7 @@
 
 
 
+### choices about user's profiles
 
 user_profiles = [
 	"helper",

solidata_api/_core/pandas_ops/__init__.py

@@ -0,0 +1 @@
+from .pd_read_files import *

solidata_api/_core/pandas_ops/pd_read_files.py

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+"""
+pd_read_files.py  
+"""
+
+
+from log_config import log, pformat
+
+log.debug("... loading pd_read_files.py ...")
+
+
+import pandas as pd
+
+
+def read_file_with_pd ( uploaded_file, file_extension ) : 
+
+	if file_extension == "csv" : 
+		df = pd.read_csv(uploaded_file)
+
+	elif file_extension in ["xls","xlsx"] : 
+		df = pd.read_excel(uploaded_file)
+
+	elif file_extension == "xml" : 
+		### TO DO !!!
+		df = pd.read_excel(uploaded_file)
+
+
+
+
+	return df 
+

solidata_api/_core/queries_db/__init__.py

@@ -25,6 +25,7 @@
 mongo_datamodels_fields 		= mongo.db[ app.config["MONGO_COLL_DATAMODELS_FIELDS"] ]
 # mongo_connectors	 					= mongo.db[ app.config["MONGO_COLL_CONNECTORS"] ]
 mongo_datasets_inputs 			= mongo.db[ app.config["MONGO_COLL_DATASETS_INPUTS"] ]
+mongo_datasets_raws 			  = mongo.db[ app.config["MONGO_COLL_DATASETS_RAWS"] ]
 mongo_recipes 							= mongo.db[ app.config["MONGO_COLL_RECIPES"] ]
 # mongo_corr_dicts 						= mongo.db[ app.config["MONGO_COLL_CORR_DICTS"] ]
 mongo_datasets_outputs 			= mongo.db[ app.config["MONGO_COLL_DATASETS_OUTPUTS"] ]
@@ -35,14 +36,16 @@
 # mongo_licences 							= mongo.db[ app.config["MONGO_COLL_LICENCES"] ]
 
 
-db = {
+db_dict = {
 					"mongo_tags"									: mongo_tags,
 					"mongo_users"									: mongo_users,
+					"mongo_users"									: mongo_users,
 					"mongo_projects"							: mongo_projects,
 					"mongo_datamodels_templates"	: mongo_datamodels_templates,
 					"mongo_datamodels_fields"			: mongo_datamodels_fields,
 					# "mongo_connectors"						: mongo_connectors,
 					"mongo_datasets_inputs"				: mongo_datasets_inputs,
+					"mongo_datasets_raws"			  	: mongo_datasets_raws,
 					"mongo_recipes"								: mongo_recipes,
 					# "mongo_corr_dicts"						: mongo_corr_dicts,
 
@@ -52,7 +55,7 @@
 			}
 
 def select_collection(coll_name):
-	coll = db[coll_name]
+	coll = db_dict[coll_name]
 	return coll
 
 

solidata_api/_core/utils/app_logs.py

@@ -9,8 +9,10 @@
 
 log.debug("... loading app_logs.py ...")
 
+from	bson.objectid import ObjectId
+from 	datetime import datetime, timedelta
+from 	solidata_api._core.queries_db import db_dict
 
-from datetime import datetime, timedelta
 
 def create_modif_log( doc, 
 											action, 
@@ -31,8 +33,32 @@ def create_modif_log( doc,
 		modif["modif_by"] = by
 
 	if val != None :
-  		modif["modif_val"] = val
+			modif["modif_val"] = val
 
 	doc["modif_log"].insert(0, modif)
 
-	return doc
+	return doc
+
+
+def add_to_datasets(coll, target_doc_oid, doc_type, oid_by, oid_to_add, include_is_fav=False) : 
+	"""
+  expects all values as already stringified
+	"""
+
+	### select mongo collection
+	mongo_coll = db_dict[coll]
+
+	### add dsi ref to user
+	doc_ = mongo_coll.find_one( {"_id" : ObjectId(target_doc_oid) } )
+
+	### create ref to add to doc datasets
+	new_ref_ 	= { "oid_"+doc_type 	: oid_to_add,
+								"added_by"				: oid_by ,
+								"added_at"				: datetime.utcnow(),
+							}
+	if include_is_fav == True :
+  		new_ref_["is_fav"]	= True
+
+	doc_["datasets"][doc_type+"_list"].append(new_ref_)
+
+	mongo_coll.save(doc_)	

solidata_api/_models/models_dataset_input.py

@@ -13,9 +13,9 @@
 from flask_restplus import fields
 
 ### import data serializers
-from solidata_api._serializers.schema_logs import *  
-from solidata_api._serializers.schema_generic import *  
-from solidata_api._serializers.schema_projects import *  
+from solidata_api._serializers.schema_logs			import *  
+from solidata_api._serializers.schema_generic		import *  
+from solidata_api._serializers.schema_projects	import *  
 
 ### import generic models functions
 from solidata_api._models.models_generic import * 
@@ -36,51 +36,50 @@ def __init__(self, ns_) :
 
 		### SELF MODULES
 		self.basic_infos 			= create_model_basic_infos(ns_, model_name="Dsi_infos")
-		self.log						 	= create_model_log(ns_, 				model_name="Dsi_log",include_is_running=True, include_is_loaded=True )
+		self.public_auth			= create_model_public_auth(ns_, model_name="Dsi_public_auth")
+		self.log						 	= create_model_log(ns_, 				model_name="Dsi_log",					include_is_running=True, include_is_loaded=True )
 		self.modif_log				= create_model_modif_log(ns_, 	model_name="Dsi_modif_log")
-		self.specs						= create_model_specs(ns_,				model_name="Dsi_specs", include_src_link=True)
+		self.specs						= create_model_specs(ns_,				model_name="Dsi_specs", 			include_src_link=True)
 		self.team 						= create_model_team(ns_,				model_name="Dsi_team")
-		self.uses							= create_uses(ns_,							model_name="Dsi_uses", 			schema_list=["usr","prj"])
-
+
+		self.uses							= create_uses(ns_,							model_name="Dsi_uses", 				schema_list=["usr","prj"])
+		self.datasets 				= create_model_datasets(ns_, 		model_name="Dsi_datasets", 		schema_list=["dsr","tag"])
+
 		### IN / complete data to enter in DB
 		self.mod_complete_in 	= ns_.model('Project_in', {
 
-				'infos' 		: self.basic_infos,
-				'specs'			: self.specs , 
-				'log'				: self.log , 
-				'modif_log'	: self.modif_log , 
-				'uses'			: self.uses,
+				'infos' 			: self.basic_infos,
+				'public_auth' : self.public_auth,
+				'specs'				: self.specs , 
+				'log'					: self.log , 
+				'modif_log'		: self.modif_log , 
 
+				### uses of the document
+				'uses'				: self.uses,
+
 				### team and edition levels
-				'team'		: self.team ,
+				'team'			: self.team ,
 
-		})
+				### datasets 
+				'datasets'	: self.datasets,
 
-		### OUT / complete data to enter in DB
-		# self.mod_complete_out  = ns_.model('Project_out', {
+		})
 
-		# 		'infos' 	: self.basic_infos,
-		# 		'log' 		: self.project_log , 
-
-		# 		### team and edition levels
-		# 		# 'project_team'					: fields.List(self.collaborator) ,
-		# 		'proj_team'		: self.collaborators ,
 
-		# 		### datasets 
-		# 		'datamodel' 						: oid,
-		# 		'dataset_inputs' 				: fields.List(oid),
-		# 		'correspondance_dicts' 	: fields.List(oid),
-		# 		'recipes' 							: fields.List(oid),
+		### IN / complete data to enter in DB
+		self.mod_minimum 	= ns_.model('Project_minimum', {
 
-		# 	})
+				'infos' 			: self.basic_infos,
+				'public_auth' : self.public_auth,
 
+		})
 
 	@property
 	def model_complete_in(self): 
 		return self.mod_complete_in
 
-	# @property
-	# def model_complete_out(self): 
-	# 	return self.mod_complete_out
 
+	@property
+	def model_minimum(self): 
+		return self.mod_minimum