# -*- coding=utf-8 -*-
#curl公共程序 pycurlcomm.py
#张明伟 20200331
import pycurl
from io import BytesIO
import datetime,time,os
import logging
import json
from logging import handlers
testdebug=1 #测试模式：1为是，0为否
if testdebug:
ip = '221.122.125.207' #测试环境IP
else:
ip = '221.122.125.201' #正式环境IP
pth=os.getcwd()+os.sep+'data'+os.sep
logpath =os.getcwd() + os.sep + 'logs' + os.sep
#返回当前日期
def dtnow():
return datetime.datetime.today().strftime('%Y%m%d')
#返回文件名相关信息
def filestr(filename):
filepath,tmpfile=os.path.split(filename)
basename,extention=os.path.splitext(tmpfile)
return filepath,tmpfile,basename,extention
#暂时不用
def init():
dt=(datetime.datetime.today()+datetime.timedelta(days=-1)).strftime('%Y-%m-%d')
rdt=dt.replace('-','')
logfile='bigdata'+rdt+'.log'
logging.basicConfig(level=logging.INFO,format='%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s',filename=logfile)
logging.info("系统金额我发你")
print(os.path.exists(logfile))
#curlcmd用于上传下载，scp文件，http请求
#op=download/upload 下载或上传 默认是下载，上传时需要传入filename,http请求时需要data
def curlcmd(url,op="download",filename=None,data=""):
if not filename:
filename=''
c = pycurl.Curl()
c.setopt(pycurl.SSL_VERIFYPEER, False) #ssl不信任或无证书时需要设置这个 等同于curl -k/--insecure 参数
c.setopt(pycurl.SSL_VERIFYHOST, False) #
c.setopt(pycurl.URL, url+filename) #上传时需要指定上传文件名，所以filename加入到url中
c.fp = BytesIO() #设置输出
c.setopt(pycurl.POST, 1) #以POST方式提交
# c.setopt(pycurl.POSTFIELDS, urllib.urlencode(data))
if isinstance(data,list): #提交大量数据
c.setopt(pycurl.HTTPPOST, data)
elif isinstance(data,dict): #json格式的数据上传 对应着大量的表单数据提交
c.setopt(pycurl.POSTFIELDS,json.dumps(data))
c.setopt(pycurl.USERPWD, "ftpuser:Cbpp@123") #用户名密码
c.setopt(pycurl.WRITEFUNCTION, c.fp.write) #curl执行结果
#print(data,url,op,filename)
if op=="upload": #上传文件
if not filename:
raise Exception("请输入上传文件名!")
filename1=pth+filename
filename1size=os.path.getsize(filename1)
c.setopt(pycurl.UPLOAD, True) #设置上传标志
c.setopt(pycurl.INFILESIZE, filename1size) #上传需要传入文件大小标志
# c.setopt(pycurl.READFUNCTION, open(filename1,"rd").read(filename1size))
c.setopt(pycurl.READFUNCTION, open(filename1, "rb").read)
# print(data, '@@@@@@')
# print(urllib.urlencode(data))
c.perform() #curl中真正的执行
code = c.getinfo(c.HTTP_CODE) #获取返回码
html = c.fp.getvalue() #获取返回结果
print(html)
return {'code':code,'html':html} #返回json格式
#日志类
class Logger(object):
level_relations={
'debug':logging.DEBUG,
'info':logging.INFO,
'waining':logging.WARNING,
'error':logging.ERROR,
'crit':logging.CRITICAL
}
def __init__(self,filename=None,level='debug',when='D',backCount=3,fmt='%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s'):
if not os.path.exists(logpath):
os.mkdir(logpath)
if filename is None:
filename=filestr(__file__)[2] #返回无后缀的文件名
logfile=logpath+filename+dtnow()+'.log'
self.logger=logging.getLogger(logfile)
format_str=logging.Formatter(fmt)
self.logger.setLevel(self.level_relations.get(level))
sh=logging.StreamHandler()
sh.setFormatter(format_str)
th=handlers.TimedRotatingFileHandler(filename=logfile,when=when,backupCount=backCount)
th.setFormatter(format_str)
self.logger.addHandler(sh)
self.logger.addHandler(th)
#json数据类型解析
class expressDict():
fldlist = [] # type:list #字段名列表
valuelist = [] # type:list #值列表
sql = "" #SQL INSERT语句列表
valuesql = "" #SQL值语句列表
def __init__(self,jsstr,sqlquote="'"):
self.jsstr = jsstr
self.sqlquote = "'"
self.tabname = ""
self.sqls=[]
self.sqlFlag = True #默认是需要得到SQL语句
# self.fldlist=[] #type:list
# self.valuelist=[] #type:list
# self.sql = ""
# self.valuesql = ""
#数据处理后再次重置数据
def __clear(self):
self.fldlist = [] # type:list
self.valuelist = [] # type:list
self.sql = ""
self.valuesql = ""
def __display_item(self, jsstr):
"""
根据json格式自动生成SQL语句或文件:递归
文件存放在{pth}目录下
sqlFlag默认为真，为真时产生sql语句，否则生成脚本
:param jsstr:
:return:
"""
#文件格式存放时不需要分割符
if not self.sqlFlag:
self.sqlquote=""
#如果json格式中数据库字典格式，则把键值做为表名
if isinstance(jsstr, dict):
for k,v in jsstr.items():
if not isinstance(v,list):
self.fldlist.append(k)
self.valuelist.append(v)
# print(k,'--->',v)
self.sql=self.sql+k+','
self.valuesql=self.valuesql+self.sqlquote+v+self.sqlquote+','
else:
if not k:
# print(self.tabname,"*" * 20)
pass
else:
if k == self.tabname:
self.sql = self.sql+') '
self.tabname=k
# print(self.tabname+'@@@@@@@')
self.__display_item(v)
# 如果json格式中数据库列表格式，则把其他数据做库数据值
elif isinstance(jsstr, list):
for jslst in jsstr:
if self.sqlFlag:
self.sql=self.sql+"insert into %s (" %(self.tabname)
self.__display_item(jslst)
self.sql = self.sql.strip(',') + ") values ("+self.valuesql.strip(',')+");"
self.sqls.append(self.sql)
self.__clear()
else:
finame = pth + self.tabname + '_' + dtnow() + '.del'
self.__display_item(jslst)
data = self.valuesql.strip(',') + os.linesep # type:str
self.__clear()
print(finame, data)
with open(finame, 'a+') as fp:
fp.write(data.encode("utf-8"))
def display_items(self,sqlFlag=True):
# return self.display_item(self.jsstr)
self.sqlFlag=sqlFlag
return self.__display_item(self.jsstr)
def dispay_sqls(self):
for sql in self.sqls:
print(sql)
# print(self.valuesql)
# curl -k -v -T "{table_name1_20200403.del,table_name2_20200403.del}" -u ftpuser scp://221.122.2.196/tmp/
# print(curlcmd(data="",url="scp://221.122.2.196//tmp/gjj20200402.log"))
# print(curlcmd(data="",url="scp://ftpdata@221.122.2.196/tmp/",op="upload",filename="test111.txt"))
