From 93001d0d6101cad5b3ea288aaba073a22848a22a Mon Sep 17 00:00:00 2001 From: loony Date: Tue, 30 Jan 2018 19:49:43 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=09=E4=BF=AE=E6=94=B9=EF=BC=9A=20=20=20=20?= =?UTF-8?q?=20createtables=5Fmysql.sql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lkad/createtables_mysql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lkad/createtables_mysql.sql b/lkad/createtables_mysql.sql index bbdbeda..5694d55 100644 --- a/lkad/createtables_mysql.sql +++ b/lkad/createtables_mysql.sql @@ -46,7 +46,7 @@ profits_yoy decimal(20, 4) comment '净利润同比(%)' , distrib varchar(50) comment '分配方案' , report_date date comment '发布日期' , year varchar(5) comment '年份' , -quarter varchar(2) '季度' +quarter varchar(2) comment '季度' ) comment '业绩报告表'; From c7ee13ac2bec1cfdd0b36691563c2a86af147297 Mon Sep 17 00:00:00 2001 From: loony Date: Tue, 30 Jan 2018 20:53:19 +0800 Subject: [PATCH 2/4] fix GetAll_mysql --- common/__pycache__/GetMysqlConn.cpython-35.pyc | Bin 969 -> 969 bytes lkad/GetAll_mysql.py | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/__pycache__/GetMysqlConn.cpython-35.pyc b/common/__pycache__/GetMysqlConn.cpython-35.pyc index 7af5c6d875d3cfde0cd68a9d474e7f44e72a9a24..43d63ebf8fafc5ab070a4cb2ec1005fee8f67e58 100644 GIT binary patch delta 105 zcmX@fev+L-jF*>7BDx@IBgcA1#;DCZ7=sx(#26SDRx%VxO%`Sjx0eBOZgIrN=O$+6 l#m8&17Ks1_inu`pABcd-=OyN*#>W>)fy5Am^yHPy3IGXP7b5@w delta 105 zcmX@fev+L-jF*?|;EOGh8#&f9GTz#}gE5$qL!5zuVI@P6^kiY?aC=!G=N3nNd~RZ9 mUVOYJYmq2Wpoj-V@Pi1Ld|qO1YJ7Z=G)N3V$V^_ztN;MQ;Tb0Y diff --git a/lkad/GetAll_mysql.py b/lkad/GetAll_mysql.py index 6954ef2..82fe6d2 100644 --- a/lkad/GetAll_mysql.py +++ b/lkad/GetAll_mysql.py @@ -78,10 +78,10 @@ def getCSV(code, url): def saveInDB(code,cursor): # 建表 if not is_table_exist(code,cursor): # 如果表不存在，先创建表 - #print("not_exist") + print(cursor,"not_exist") create_table(code,cursor) # 如果表不存在，先建表 else: # 存在则截断 - #print("exist") + print(cursor,"exist") cursor.execute("truncate table stock_" + code) cursor.execute("commit") @@ -214,7 +214,7 @@ def get_type(code): return switcher.get(code_pre, '未知') # 建表 -def create_table(code): +def create_table(code,cursor): sql = "CREATE TABLE stock_" + code + """ ( UUID VARCHAR(80) PRIMARY KEY, @@ -237,7 +237,7 @@ def create_table(code): ZBS decimal(20, 4) ) """ - #print(sql) + print(sql) cursor.execute(sql) cursor.execute('commit') # 添加注释 @@ -268,16 +268,16 @@ def create_table(code): def is_table_exist(code,cursor): table = "stock_" + code sql = " select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_NAME='%s' " %table - #print(sql) + print(sql) rs = cursor.execute(sql) result = cursor.fetchall() #print(result) if len(result) > 0: return True - #ptint(table+"True") + ptint(table+"True") else: return False - #ptint(table+"false") + ptint(table+"false") def main(key, url): From 8fe0616e082002fef763bb8314910e300943cab0 Mon Sep 17 00:00:00 2001 From: loony Date: Tue, 30 Jan 2018 20:54:26 +0800 Subject: [PATCH 3/4] fix GetAll_mysql --- lkad/requirement.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 lkad/requirement.txt diff --git a/lkad/requirement.txt b/lkad/requirement.txt new file mode 100644 index 0000000..c7b5d62 --- /dev/null +++ b/lkad/requirement.txt @@ -0,0 +1,15 @@ +beautifulsoup4==4.6.0 +bs4==0.0.1 +chardet==2.3.0 +cx-Oracle==6.1 +lxml==4.1.1 +numpy==1.14.0 +pandas==0.22.0 +PyMySQL==0.8.0 +pytz==2017.3 +requests==2.9.1 +six==1.10.0 +SQLAlchemy==1.2.2 +ssh-import-id==5.5 +tushare==1.1.3 +urllib3==1.13.1 From 1fb9001574ea697e081621e0eaddbc9cc90b222f Mon Sep 17 00:00:00 2001 From: lkad Date: Sat, 3 Feb 2018 09:31:54 +0800 Subject: [PATCH 4/4] add reade.text --- .../__pycache__/GetMysqlConn.cpython-35.pyc | Bin 969 -> 969 bytes lkad/GetAllInc_mysql.py | 5 ++- lkad/GetAll_mysql.py | 39 ++++++++++-------- lkad/reade.text | 8 ++++ lkad/requirement.txt | 4 +- 5 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 lkad/reade.text diff --git a/common/__pycache__/GetMysqlConn.cpython-35.pyc b/common/__pycache__/GetMysqlConn.cpython-35.pyc index 43d63ebf8fafc5ab070a4cb2ec1005fee8f67e58..a9b099a3db0e72f87f817880475256bed51e1f99 100644 GIT binary patch delta 103 zcmX@fev+L{jF*>7K4&A_Mn=Y4n|Cky=r%*>0A j*JLdc1qu}LfCzpN0h7;5%uS7tFOmj{AqbhtE14Al@BJ3~ delta 103 zcmX@fev+L{jF*>7B6=g+Mn=Y{%{v%_89Brl7#LPE6iH1MW)8QP0dj6}#K-3*X6D7m jYqA!J00oM;K?EO&fXU}2=BCES7fFG{5QOyPmCOnNx&{_k diff --git a/lkad/GetAllInc_mysql.py b/lkad/GetAllInc_mysql.py index 2ebe175..e41249e 100644 --- a/lkad/GetAllInc_mysql.py +++ b/lkad/GetAllInc_mysql.py @@ -51,6 +51,7 @@ def getTable(): maxDateList = [] for i in tableNameList: sqlStr = "SELECT DATE_FORMAT(max(DATE),'%Y%m%d') as maxdate FROM "+ i + print(sqlStr) cursor.execute(sqlStr) maxDate = cursor.fetchone()["maxdate"] @@ -79,7 +80,7 @@ def getCSV(code, url): fordername = 'AllStockDataInc' filename = str(code) + '.CSV' if not os.path.isdir(fordername): - print("mkdir") + #print("mkdir") os.mkdir(fordername) with request.urlopen(url) as web: @@ -199,7 +200,7 @@ def main(): # print(e #获取下载链接和股票代码 valid_code_url=return_valid_code_url() - pool=Pool(processes=24) + pool=Pool(processes=3) for c,u in valid_code_url: pool.apply_async(getCSV,(c,u,)) diff --git a/lkad/GetAll_mysql.py b/lkad/GetAll_mysql.py index 82fe6d2..34fc414 100644 --- a/lkad/GetAll_mysql.py +++ b/lkad/GetAll_mysql.py @@ -52,15 +52,18 @@ def listStock(cursor): def getCSV(code, url): fordername = 'AllStockData' filename = str(code) + '.CSV' + fullfilename = fordername+os.path.sep+filename if not os.path.isdir(fordername): - print("mkdir") + #print("mkdir") os.mkdir(fordername) - with request.urlopen(url) as web: - # 为防止编码错误，使用二进制写文件模式 - print(web) - with open(fordername+os.path.sep+filename, 'wb') as outfile: - outfile.write(web.read()) - print("write OK "+str(code)) + if not os.path.exists(fullfilename): + with request.urlopen(url) as web: + # 为防止编码错误，使用二进制写文件模式 + # print(web) + with open(fullfilename, 'wb') as outfile: + outfile.write(web.read()) + print("csvfile download OK "+str(code)) + #print("id1") #print(id(cursor)) @@ -69,19 +72,19 @@ def getCSV(code, url): #print("Id2") # print(id(cursor)) saveInDB(code,cursor) - - print("一只股票入库完毕") - # 删除CSV文件 - os.remove(fordername+filename) + cursor.close() + print("一只股票入库完毕",code) + # 删除CSV文件,如果需要保留，直接注释即可，适合网络状态不太好的时候。 + os.remove(fullfilename) # 将获取的数据入库 def saveInDB(code,cursor): # 建表 if not is_table_exist(code,cursor): # 如果表不存在，先创建表 - print(cursor,"not_exist") + #print(cursor,"not_exist") create_table(code,cursor) # 如果表不存在，先建表 else: # 存在则截断 - print(cursor,"exist") + #print(cursor,"exist") cursor.execute("truncate table stock_" + code) cursor.execute("commit") @@ -237,7 +240,7 @@ def create_table(code,cursor): ZBS decimal(20, 4) ) """ - print(sql) + #print(sql) cursor.execute(sql) cursor.execute('commit') # 添加注释 @@ -268,16 +271,16 @@ def create_table(code,cursor): def is_table_exist(code,cursor): table = "stock_" + code sql = " select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_NAME='%s' " %table - print(sql) + #print(sql) rs = cursor.execute(sql) result = cursor.fetchall() #print(result) if len(result) > 0: return True - ptint(table+"True") + #ptint(table+"True") else: return False - ptint(table+"false") + #ptint(table+"false") def main(key, url): @@ -289,7 +292,7 @@ def main(key, url): #======================================================= time1 = time.time() dict = listStock(cursor) - pool = Pool(processes = 24) # 设定并发进程的数量 + pool = Pool(processes = 10) # 设定并发进程的数量 for key in dict: pool.apply_async(main, (key, dict[key], )) diff --git a/lkad/reade.text b/lkad/reade.text new file mode 100644 index 0000000..765e5ce --- /dev/null +++ b/lkad/reade.text @@ -0,0 +1,8 @@ +使用说明请看这里： + lkad下面是mysql数据库的代码，使用方法如下： + 安装依赖 pip3 install -r requirement.txt + 然后运行建表语句：mysql -uroot -P