diff --git a/common/__pycache__/GetMysqlConn.cpython-35.pyc b/common/__pycache__/GetMysqlConn.cpython-35.pyc index 7af5c6d..a9b099a 100644 Binary files a/common/__pycache__/GetMysqlConn.cpython-35.pyc and b/common/__pycache__/GetMysqlConn.cpython-35.pyc differ diff --git a/lkad/GetAllInc_mysql.py b/lkad/GetAllInc_mysql.py index 2ebe175..e41249e 100644 --- a/lkad/GetAllInc_mysql.py +++ b/lkad/GetAllInc_mysql.py @@ -51,6 +51,7 @@ def getTable(): maxDateList = [] for i in tableNameList: sqlStr = "SELECT DATE_FORMAT(max(DATE),'%Y%m%d') as maxdate FROM "+ i + print(sqlStr) cursor.execute(sqlStr) maxDate = cursor.fetchone()["maxdate"] @@ -79,7 +80,7 @@ def getCSV(code, url): fordername = 'AllStockDataInc' filename = str(code) + '.CSV' if not os.path.isdir(fordername): - print("mkdir") + #print("mkdir") os.mkdir(fordername) with request.urlopen(url) as web: @@ -199,7 +200,7 @@ def main(): # print(e #获取下载链接和股票代码 valid_code_url=return_valid_code_url() - pool=Pool(processes=24) + pool=Pool(processes=3) for c,u in valid_code_url: pool.apply_async(getCSV,(c,u,)) diff --git a/lkad/GetAll_mysql.py b/lkad/GetAll_mysql.py index 6954ef2..34fc414 100644 --- a/lkad/GetAll_mysql.py +++ b/lkad/GetAll_mysql.py @@ -52,15 +52,18 @@ def listStock(cursor): def getCSV(code, url): fordername = 'AllStockData' filename = str(code) + '.CSV' + fullfilename = fordername+os.path.sep+filename if not os.path.isdir(fordername): - print("mkdir") + #print("mkdir") os.mkdir(fordername) - with request.urlopen(url) as web: - # 为防止编码错误，使用二进制写文件模式 - print(web) - with open(fordername+os.path.sep+filename, 'wb') as outfile: - outfile.write(web.read()) - print("write OK "+str(code)) + if not os.path.exists(fullfilename): + with request.urlopen(url) as web: + # 为防止编码错误，使用二进制写文件模式 + # print(web) + with open(fullfilename, 'wb') as outfile: + outfile.write(web.read()) + print("csvfile download OK "+str(code)) + #print("id1") #print(id(cursor)) @@ -69,19 +72,19 @@ def getCSV(code, url): #print("Id2") # print(id(cursor)) saveInDB(code,cursor) - - print("一只股票入库完毕") - # 删除CSV文件 - os.remove(fordername+filename) + cursor.close() + print("一只股票入库完毕",code) + # 删除CSV文件,如果需要保留，直接注释即可，适合网络状态不太好的时候。 + os.remove(fullfilename) # 将获取的数据入库 def saveInDB(code,cursor): # 建表 if not is_table_exist(code,cursor): # 如果表不存在，先创建表 - #print("not_exist") + #print(cursor,"not_exist") create_table(code,cursor) # 如果表不存在，先建表 else: # 存在则截断 - #print("exist") + #print(cursor,"exist") cursor.execute("truncate table stock_" + code) cursor.execute("commit") @@ -214,7 +217,7 @@ def get_type(code): return switcher.get(code_pre, '未知') # 建表 -def create_table(code): +def create_table(code,cursor): sql = "CREATE TABLE stock_" + code + """ ( UUID VARCHAR(80) PRIMARY KEY, @@ -289,7 +292,7 @@ def main(key, url): #======================================================= time1 = time.time() dict = listStock(cursor) - pool = Pool(processes = 24) # 设定并发进程的数量 + pool = Pool(processes = 10) # 设定并发进程的数量 for key in dict: pool.apply_async(main, (key, dict[key], )) diff --git a/lkad/createtables_mysql.sql b/lkad/createtables_mysql.sql index bbdbeda..5694d55 100644 --- a/lkad/createtables_mysql.sql +++ b/lkad/createtables_mysql.sql @@ -46,7 +46,7 @@ profits_yoy decimal(20, 4) comment '净利润同比(%)' , distrib varchar(50) comment '分配方案' , report_date date comment '发布日期' , year varchar(5) comment '年份' , -quarter varchar(2) '季度' +quarter varchar(2) comment '季度' ) comment '业绩报告表'; diff --git a/lkad/reade.text b/lkad/reade.text new file mode 100644 index 0000000..765e5ce --- /dev/null +++ b/lkad/reade.text @@ -0,0 +1,8 @@ +使用说明请看这里： + lkad下面是mysql数据库的代码，使用方法如下： + 安装依赖 pip3 install -r requirement.txt + 然后运行建表语句：mysql -uroot -P