diff --git a/PROXIES/db.conf b/PROXIES/db.conf new file mode 100644 index 0000000..1066d95 --- /dev/null +++ b/PROXIES/db.conf @@ -0,0 +1,31 @@ +[mysql] + +HOST = 172.20.6.100 +PORT = 3306 +USER = root +PASSWD = mysqladmin +DB = pydb +TABLE = pytab +CHARSET = utf8 + + +[redis] + +HOST = 172.20.6.100 +PORT = 6379 +PASSWD = redisadmin + + +[memcache] + +HOST = 172.20.6.100 +PORT = 11211 + + +[mongodb] + +HOST = 172.20.6.100 +PORT = 27017 +DB = db1 +USER = mongoadmin +PASSWD = mongopwd diff --git a/PROXIES/save_memcache.py b/PROXIES/save_memcache.py new file mode 100644 index 0000000..8edf889 --- /dev/null +++ b/PROXIES/save_memcache.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import memcache +import random +import configparser +import spider + +class MemcacheOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['memcache']['HOST'] + self.port = config['memcache']['PORT'] + self.mcoper = memcache.Client([self.host+':'+self.port], debug = True) + + def memcache_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.mcoper.set(str(num),cont) + + def memcache_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.mcoper.get(str(num)) + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MemcacheOper() + dbhelper.memcache_save(proxy_ip) + ip = dbhelper.memcache_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mongodb.py b/PROXIES/save_mongodb.py new file mode 100644 index 0000000..0f76994 --- /dev/null +++ b/PROXIES/save_mongodb.py @@ -0,0 +1,59 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + + + +import configparser +import spider +from pymongo import MongoClient + +class MongodbOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mongodb']['HOST'] + self.port = config['mongodb']['PORT'] + self.db = config['mongodb']['DB'] + self.user = config['mongodb']['USER'] + self.pwd = config['mongodb']['PASSWD'] + self.client = MongoClient(self.host, int(self.port)) + self.db_auth = self.client.admin + self.db_auth.authenticate(self.user,self.pwd) + self.DB = self.client[self.db] + self.collection = self.DB.myset + + def mongodb_save(self,result_list): + """ + save data + :return:None + """ + + for values in result_list: + self.collection.insert(values) + + def mongodb_gain(self): + """ + gain data + :return: proxies + """ + ip = self.collection.find_one() + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MongodbOper() + dbhelper.mongodb_save(proxy_ip) + ip = dbhelper.mongodb_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mysql.py b/PROXIES/save_mysql.py new file mode 100644 index 0000000..f7c40c7 --- /dev/null +++ b/PROXIES/save_mysql.py @@ -0,0 +1,64 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import pymysql +import configparser +import spider + + +class MysqlOper: + # initial database information + def __init__(self, result_list): + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mysql']['HOST'] + self.port = int(config['mysql']['PORT']) + self.user = config['mysql']['USER'] + self.passwd = config['mysql']['PASSWD'] + self.db = config['mysql']['DB'] + self.table = config['mysql']['TABLE'] + self.charset = config['mysql']['CHARSET'] + self.result_list = result_list + + def mysql_save(self): + + # create db cursor + try: + DB = pymysql.connect(self.host, self.user, self.passwd, self.db, port=self.port, charset=self.charset) + cursor = DB.cursor() + except Exception as e: + print("connect dbserver fail,Please see information:") + print(e) + exit(1) + + # check and create tables + cursor.execute('show tables in pydb') + tables = cursor.fetchall() + flag = True + for tab in tables: + if self.table in tab: + flag = False + print('%s is exist' % self.table) + print(flag) + if flag: + cursor.execute( + '''create table pytab (id int unsigned not null primary key auto_increment, protocol varchar(10),content varchar(50))''') + else: + return 0 + + # write database + for values in self.result_list: + for prot, cont in values.items(): + try: + cursor.execute("insert into pytab (protocol,content) value (%s,%s);", [prot, cont]) + except Exception as e: + print("insert db occer error", e) + + +if __name__ == "__main__": + proxyhelper = spider.GetProxyIP(3) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MysqlOper(proxy_ip) + dbhelper.mysql_save() diff --git a/PROXIES/save_redis.py b/PROXIES/save_redis.py new file mode 100644 index 0000000..5c7bb91 --- /dev/null +++ b/PROXIES/save_redis.py @@ -0,0 +1,52 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import redis +import random +import configparser +import spider + +class RedisOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['redis']['HOST'] + self.port = config['redis']['PORT'] + self.passwd = config['redis']['PASSWD'] + self.pool = redis.ConnectionPool(host=self.host,port=self.port,password=self.passwd) + self.redis_helper = redis.Redis(connection_pool=self.pool) + self.pipe = self.redis_helper.pipeline(transaction=True) + + def redis_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.redis_helper.set(num,cont) + self.pipe.execute() + + def redis_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.redis_helper.get(num) + self.pipe.execute() + return ip + +if __name__ == '__main__': + # proxyhelper = spider.GetProxyIP(2) + # res_pool = proxyhelper.get_ip() + # proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = RedisOper() + # dbhelper.redis_save(proxy_ip) + ip = dbhelper.redis_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/spider.py b/PROXIES/spider.py new file mode 100644 index 0000000..9b538c6 --- /dev/null +++ b/PROXIES/spider.py @@ -0,0 +1,61 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import requests +from bs4 import BeautifulSoup +import random + +class GetProxyIP: + + def __init__(self,page=10): + self._page = page + self.url_head = 'http://www.xicidaili.com/wt/' + + def get_ip(self): + """ + get resouce proxy ip pool + :return: res_pool list + """ + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"} + res_pool = [] + for pagenum in range(1,self._page): + url = self.url_head + str(pagenum) + response = requests.get(url, headers=headers) + soup = BeautifulSoup(response.text, "html.parser") + soup_tr = soup.find_all('tr') + for item in soup_tr: + try: + soup_td = item.find_all('td') + res_pool.append(soup_td[5].text.lower() + '://' + soup_td[1].text + ':' + soup_td[2].text) + except IndexError: + pass + return res_pool + + def right_proxies(self,res_pool): + """ + check available ip + :param res_pool: + :return:right_pool list + """ + right_pool = [] + for ip in res_pool: + if 'https' in ip: + proxies = {'http': ip} + else: + proxies = {"http": ip} + check_urllist = ['http://www.baidu.com', 'http://www.taobao.com', 'https://cloud.tencent.com/'] + try: + response = requests.get(random.choice(check_urllist), proxies=proxies, timeout = 1) + if response.status_code: + right_pool.append(proxies) + print('add ip %s' % proxies) + except Exception as e: + continue + return right_pool + +if __name__ == '__main__': + proxyhelper = GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip =proxyhelper.right_proxies(res_pool) + print(proxy_ip) diff --git a/RDSBAK/Rds_backup.py b/RDSBAK/Rds_backup.py new file mode 100644 index 0000000..d01cddc --- /dev/null +++ b/RDSBAK/Rds_backup.py @@ -0,0 +1,69 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from aliyunsdkcore import client +from aliyunsdkrds.request.v20140815 import CreateBackupRequest +import time +import os +import logging + +class rdsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('LTAIhfXlcjyl****','GwfAM344K2ELm345184356TVgRfAso','cn-shanghai') + self.logger = logger + self.logger.info("------------------------start exec rds backup API log-------------") + def backup_instance(self): + # 设置参数 + request = CreateBackupRequest.CreateBackupRequest() + request.set_accept_format('json') + request.add_query_param('DBInstanceId', 'rm-uf6x**5u1x842y61y') + + #如果为单库备份,可以添加DBName + # request.add_query_param('DBName', 'mydb') + + #BackupMethod为备份方式:Logical:逻辑备份,Physical:物理备份 + request.add_query_param('BackupMethod', 'Physical') + #BackupType为备份类型: Auto:自动计算是全量备份还是增量备份;FullBackup:全量备份。默认值为Auto。 + request.add_query_param('BackupType', 'Auto') + + response = self.clentoper.do_action_with_exception(request) + self.logger.info("rdsbackup mission submission successful!") + self.logger.info(response) + print(response) + + +class Rds_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'rdsbackuplog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Rds_Log('rdsbackup.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = rdsOper(logger) + app.backup_instance() + + diff --git a/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log new file mode 100644 index 0000000..02ab8ae --- /dev/null +++ b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log @@ -0,0 +1,3 @@ +2018-06-04 15:40:41,364 - root - INFO - ------------------------start exec rds backup API log------------- +2018-06-04 15:40:41,767 - root - INFO - rdsbackup mission submission successful! +2018-06-04 15:40:41,767 - root - INFO - b'{"RequestId":"2AA53A09-33D3-4A26-94D3-A745301E3A02"}' diff --git a/README.md b/README.md index 082ea2b..6715d4a 100644 --- a/README.md +++ b/README.md @@ -1 +1,66 @@ # my-python-code +## 重启深信服(SANGFOROPER) +> Python深信服重启 + +blog:[Python重启深信服设备](http://blog.51cto.com/kaliarch/2095178) + +## 翻译小工具(FANYI) +> Python实现翻译小工具 + +blog:[Python实现翻译小工具](http://blog.51cto.com/kaliarch/2072150) + +## 汽车票查询(XAGLKP) +> python搜索汽车票 + +blog:[python搜索汽车票](http://blog.51cto.com/kaliarch/2071288) + +## 电影爬取(DYTT8) +> 爬取电影并存储到excel + +blog:[爬取搜索出来的电影的下载地址并保存到excel](http://blog.51cto.com/kaliarch/2069544) + +## 爬取推荐博客(51BLOG) +> 爬取推进博客 + +blog:[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto.com/kaliarch/2067103) + +## 构建自己的代理库(PROXIES) +> Python构建自己的代理库 + +blog:[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) + + +## Python自定义阿里云RDS备份策略(RDSBAK) +> Python自定义阿里云RDS备份策略 + +blog:[Python自定义阿里云RDS备份策略](http://blog.51cto.com/kaliarch/2124609) + +## RDS物理备份恢复至线下IDC服务器实践(RDSBAK) +> RDS物理备份恢复至线下IDC服务器实践 + +blog:[RDS物理备份自动恢复至IDC线下服务器实践](#) + +## Python实现端口检测(tcp_port_check) +> Python实现端口检测 + +blog:[Python实现端口检测](http://blog.51cto.com/kaliarch/2149228) + +## Python实现ECS自动镜像创建&镜像复制至其他地域(imageoper) +> Python实现ECS自动镜像创建&镜像复制至其他地域 + +blog:[Python实现ECS自动镜像创建&镜像复制至其他地域](http://blog.51cto.com/kaliarch/2150076) + +## Python多线程爬图&Scrapy框架爬图(doutulai) +> Python多线程爬图&Scrapy框架爬图 + +blog:[Python多线程爬图&Scrapy框架爬图](http://blog.51cto.com/kaliarch/2162411) + +## Python操作CVM(cvmoper) +> Python操作CVM + +blog:[Python操作CVM](http://blog.51cto.com/kaliarch/2165000) + +## 利用Python批量保存51CTO博客 +> 利用Python批量保存51CTO博客 + +blog:[利用Python批量保存51CTO博客](http://blog.51cto.com/kaliarch/2301359) diff --git a/SANGFOROPER/Check_scripts.sh b/SANGFOROPER/Check_scripts.sh new file mode 100644 index 0000000..a31edf8 --- /dev/null +++ b/SANGFOROPER/Check_scripts.sh @@ -0,0 +1,8 @@ +#!/bin/bash +num=$(ps -ef |grep pdc.sh|wc -l) +cmd="/usr/bin/nohup /bin/bash /sangfor/Shscripts/pdc/Ping_check.sh &" + +if [ ${num} -lt 2 ];then +${cmd} +fi + diff --git a/SANGFOROPER/Ecs_apireboot.py b/SANGFOROPER/Ecs_apireboot.py new file mode 100644 index 0000000..a6610c2 --- /dev/null +++ b/SANGFOROPER/Ecs_apireboot.py @@ -0,0 +1,81 @@ +#!/bin/env python3 +# -*- coding:UTF-8 -*- +# _author:kaliarch +from aliyunsdkcore import client +from aliyunsdkecs.request.v20140526 import RebootInstanceRequest,StartInstanceRequest,StopInstanceRequest +import time +import os +import logging +class ecsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('', '', 'cn-hangzhou') + self.logger = logger + self.logger.info("------------------------start reboot vpn ecs of API log-------------") + def reboot_instance(self): + # 设置参数 + request = RebootInstanceRequest.RebootInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response) + print(response) + + def start_instance(self): + request = StartInstanceRequest.StartInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn start successful!") + self.logger.info(response) + print(response) + + def stop_instance(self): + request = StopInstanceRequest.StopInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + request.add_query_param('ForceStop', 'false') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info(response) + print(response) + + def testlog(self): + self.logger.info("public test log") + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = ecsOper(logger) + app.reboot_instance() + diff --git a/SANGFOROPER/Ping_check.sh b/SANGFOROPER/Ping_check.sh new file mode 100644 index 0000000..7ce9231 --- /dev/null +++ b/SANGFOROPER/Ping_check.sh @@ -0,0 +1,42 @@ +#!/bin/bash +IP=10.10.10.2 +dir="/sangfor/Shscripts/pdc/" +if [ ! -d ${dir} ];then + mkdir -p ${dir} +fi +echo 1 > ${dir}pdcping.lock +while true +do + Time=`date +%F` + TIME="${Time} 23:59" + if [ "${data}" == "${TIME}" ];then + mkdir ${dir}${Time} && mv ${dir}pdcping.log ${dir}${Time}-pingpdc.log + mv ${dir}${Time}-pingpdc.log ${dir}${Time} + fi + find ${dir} -mtime +7 -type d -exec rm -rf {} \; + + find ${dir} -mtime +7 -name "*-pingpdc.log" -exec rm -rf {} \; + data=`date +%F' '%H:%M` + data1=`date +%F' '%H:%M:%S` + echo "------------${data1}---------------">>${dir}pingpdc.log + ping -c 10 ${IP} >>${dir}pingpdc.log + if [ $? -eq 1 ];then + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 1 ];then + /usr/local/python34/bin/python3 /sangfor/Pysangfor/SangFor_check.py + echo 0 > ${dir}pdcping.lock + else + continue + fi + else + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 0 ];then + echo 1 > ${dir}pdcping.lock + else + continue + fi + fi + + +done + diff --git a/SANGFOROPER/SangFor_oper.py b/SANGFOROPER/SangFor_oper.py new file mode 100644 index 0000000..cbfb72a --- /dev/null +++ b/SANGFOROPER/SangFor_oper.py @@ -0,0 +1,90 @@ +#!/bin/env python3 +from pyvirtualdisplay import Display +from selenium import webdriver +import time +import os +import logging + +class Glp_SangFor: + def __init__(self,logger): + self.logger = logger + self.logger.info("--------------start log----------------") + self.display = Display(visible=0, size=(800, 600)) + self.display.start() + self.browser = webdriver.Firefox() + self.logger.info("start browser successfuly") + self.sangfor_url = "深信服url" + self.username = '深信服用户名' + self.password = '深信服密码' + + def login(self): + self.browser.get(self.sangfor_url) + self.browser.implicitly_wait(5) + self.browser.find_element_by_name('user').send_keys(self.username) + self.browser.find_element_by_name('password').send_keys(self.password) + self.browser.find_element_by_class_name('buttons').click() + self.browser.implicitly_wait(5) + self.logger.info("loggin sangfor successfuly") + + def client_reboot(self): + self.browser.find_element_by_id("ext-gen111").click() + print(self.browser.find_element_by_id("ext-gen111").text) + self.browser.implicitly_wait(15) + time.sleep(60) + self.logger.info("switch mainiframe start") + try: + print(self.browser.find_element_by_link_text("重启/重启服务/关机").text) + self.browser.find_element_by_link_text("重启/重启服务/关机").click() + self.browser.implicitly_wait(3) + self.browser.switch_to_frame("mainiframe") + self.browser.implicitly_wait(8) + time.sleep(10) + self.browser.find_element_by_xpath("//button[@id='ext-gen19']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen19']").text) + self.browser.implicitly_wait(10) + #self.browser.find_element_by_xpath("//button[@id='ext-gen42']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen42']").text) + except Exception as e: + self.logger.exception("reboot successful") + return 1 + self.browser.close() + self.logger.info("browser close successful") + self.logger.info("--------------end log----------------") + return 0 + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == '__main__': + os.system("pkill firefox") + os.system("pkill Xvfb") + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + sangfor_oper = Glp_SangFor(logger) + sangfor_oper.login() + sangfor_oper.client_reboot() diff --git a/cvmoper/config.py b/cvmoper/config.py new file mode 100644 index 0000000..4f8bc60 --- /dev/null +++ b/cvmoper/config.py @@ -0,0 +1,16 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +[common] +# 腾讯云secretid +SecretId = AKIDjPYbTBU4FF4xxxxxxxxxxxxxxxxxxxxxx +# 腾讯云secretkey +SecretKey = e7RaXYVP63rUvBNUQF1iKxxxxxxxx +# cvm 所在地域,可参考https://cloud.tencent.com/document/api/213/15708 +Region = ap-shanghai + +# 腾讯云cvm实例id,多个用,隔开 +InstanceIds = ins-h8d3exxx + + diff --git a/cvmoper/cvm_oper.py b/cvmoper/cvm_oper.py new file mode 100644 index 0000000..bbb8cd8 --- /dev/null +++ b/cvmoper/cvm_oper.py @@ -0,0 +1,104 @@ +#!/bin/env python3 +# -*- coding:UTF-8 -*- +# _author:kaliarch +from tencentcloud.common import credential +from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException +# 导入对应产品模块的client models。 +from tencentcloud.cvm.v20170312 import cvm_client, models +import configparser +import time +import os +import logging +class CvmOper(): + def __init__(self,logger): + config = configparser.ConfigParser() + config.read('config.py',encoding='utf-8') + self.instance_list = config['common']['InstanceIds'].split(',') + print(self.instance_list) + cred = credential.Credential(config['common']['SecretId'], config['common']['SecretKey']) + self.clentoper = cvm_client.CvmClient(cred, config['common']['Region']) + + self.logger = logger + self.logger.info("------------------------start cvm of API log-------------") + def reboot_instance(self): + """ + 重启cvm + :return: + """ + # 设置参数 + request = models.RebootInstancesRequest() + request.InstanceIds=self.instance_list + # 发起请求 + response = self.clentoper.RebootInstances(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response.to_json_string()) + print(response.to_json_string()) + + def start_instance(self): + """ + 启动cvm + :return: + """ + request = models.StartInstancesRequest() + request.InstanceIds = self.instance_list + # 发起请求 + response = self.clentoper.StartInstances(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response.to_json_string()) + print(response.to_json_string()) + + def stop_instance(self): + """ + 停止cvm + :return: + """ + request = models.StopInstancesRequest() + request.InstanceIds = self.instance_list + # 发起请求 + response = self.clentoper.StopInstances(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response.to_json_string()) + print(response.to_json_string()) + + def testlog(self): + self.logger.info("public test log") + +class CvmLog: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'cvmlog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + cvmloger = CvmLog('cvm-oper.log') + logfilename = cvmloger.createDir() + logger = cvmloger.createlogger(logfilename) + # 生成app对象 + app = CvmOper(logger) + # 启动服务 + # app.start_instance() + # 停止实例 + # app.stop_instance() + # 重启实例 + app.reboot_instance() + diff --git a/cvmoper/cvmlog/2018-08-27-cvm-oper.log b/cvmoper/cvmlog/2018-08-27-cvm-oper.log new file mode 100644 index 0000000..b2a67a7 --- /dev/null +++ b/cvmoper/cvmlog/2018-08-27-cvm-oper.log @@ -0,0 +1,6 @@ +2018-08-27 16:35:34,533 - root - INFO - ------------------------start cvm of API log------------- +2018-08-27 16:35:35,092 - root - INFO - cvm [u'ins-h8d3e69v'] reboot successful! +2018-08-27 16:35:35,092 - root - INFO - {"RequestId": "17840c5f-65a9-41a1-9ab6-9e03e55d286b"} +2018-08-27 16:37:25,822 - root - INFO - ------------------------start cvm of API log------------- +2018-08-27 16:37:26,433 - root - INFO - cvm [u'ins-h8d3e69v'] reboot successful! +2018-08-27 16:37:26,434 - root - INFO - {"RequestId": "fe987e97-a80d-4a9a-ba6b-19b88b069d93"} diff --git a/doutulai/multithreading_spider/dutulai_spider.py b/doutulai/multithreading_spider/dutulai_spider.py new file mode 100644 index 0000000..d990548 --- /dev/null +++ b/doutulai/multithreading_spider/dutulai_spider.py @@ -0,0 +1,117 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +from bs4 import BeautifulSoup +import urllib +import os +import threading + +class DutuSpider(): + # 定义全局页面url列表 + page_url_list = [] + # 定义具体各表情图片url列表 + img_url_list = [] + # 定义rlock进程锁 + rlock = threading.RLock() + + def __init__(self,page_number=10,img_dir='imgdir',thread_number=5): + """ + :param page_number: 抓去多少个页面,默认10 + :param img_dir: 定义图片目录 + :param thread_number:默认5个线程 + """ + self.spider_url = 'https://www.doutula.com/photo/list/?page=' + self.page_number = int(page_number) + self.img_dir = img_dir + self.thread_num = thread_number + + + def get_url(self): + """ + 创建image目录和生产pageurl列表 + :return: + """ + if not os.path.exists(self.img_dir): + os.makedirs(self.img_dir) + for page in range(1,self.page_number+1): + DutuSpider.page_url_list.append(self.spider_url + str(page)) + + def __set_header(self): + """ + 定义header + :return: + """ + header = { + 'Upgrade-Insecure-Requests': '1', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36', + } + return header + + def __add_urllist(self): + """ + 定义从page_url_list 爬取具体的image的url + :return: + """ + while True: + DutuSpider.rlock.acquire() + if len(DutuSpider.page_url_list) == 0: + DutuSpider.rlock.release() + break + else: + page_url = DutuSpider.page_url_list.pop() + DutuSpider.rlock.release() + response = requests.get(page_url, headers=self.__set_header()) + soup = BeautifulSoup(response.content,'lxml') + sou_list = soup.find_all('img',attrs={'class':'img-responsive lazy image_dta'}) + # 将获取到的具体表情图标的url保存添加进img_url_list 列表 + for url_content in sou_list: + DutuSpider.rlock.acquire() + DutuSpider.img_url_list.append(url_content['data-original']) + DutuSpider.rlock.release() + + + + def __download_img(self): + """ + 从image_url_list中来下载image到本地 + :return: + """ + while True: + DutuSpider.rlock.acquire() + if len(DutuSpider.img_url_list) == 0: + DutuSpider.rlock.release() + continue + else: + img_url = DutuSpider.img_url_list.pop() + DutuSpider.rlock.release() + try: + # 图片名称 + img_name = img_url.split('/')[-1] + # 下载图片 + urllib.urlretrieve(img_url,os.path.join(self.img_dir,img_name)) + print('donload img %s' % img_name) + except Exception as e: + pass + + + def run(self): + # 启动thread_num个进程来爬去具体的img url 链接 + for th in range(self.thread_num): + add_pic_t = threading.Thread(target=self.__add_urllist) + add_pic_t.start() + + # 启动thread_num个来下载图片 + for img_th in range(self.thread_num): + download_t = threading.Thread(target=self.__download_img) + download_t.start() + + + +if __name__ == '__main__': + spider = DutuSpider(page_number=10,img_dir='imgdir',thread_number=7) + spider.get_url() + spider.run() + + diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmt6e176j205605cglx.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmt6e176j205605cglx.jpg new file mode 100644 index 0000000..9473097 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmt6e176j205605cglx.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtaqrrgj205i05gweg.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtaqrrgj205i05gweg.jpg new file mode 100644 index 0000000..7ce780b Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtaqrrgj205i05gweg.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtf4ctcj20730730sp.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtf4ctcj20730730sp.jpg new file mode 100644 index 0000000..d72f8f0 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtf4ctcj20730730sp.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtjiovoj209q09qt8v.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtjiovoj209q09qt8v.jpg new file mode 100644 index 0000000..927d4f9 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtjiovoj209q09qt8v.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtotth4j20a00a0aak.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtotth4j20a00a0aak.jpg new file mode 100644 index 0000000..494d58f Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtotth4j20a00a0aak.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtyvr48j20m80cut9o.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtyvr48j20m80cut9o.jpg new file mode 100644 index 0000000..cb4e691 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmtyvr48j20m80cut9o.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmu3h5u3j204n05iwef.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmu3h5u3j204n05iwef.jpg new file mode 100644 index 0000000..561a6ef Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmu3h5u3j204n05iwef.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmudzn2ij20k00k00ua.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmudzn2ij20k00k00ua.jpg new file mode 100644 index 0000000..214ec07 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmudzn2ij20k00k00ua.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuj3kaxj2079075t8s.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuj3kaxj2079075t8s.jpg new file mode 100644 index 0000000..8476f51 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuj3kaxj2079075t8s.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmunk5q8j208c07ywef.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmunk5q8j208c07ywef.jpg new file mode 100644 index 0000000..febeec5 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmunk5q8j208c07ywef.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmusbtmeg207i07iaac.gif b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmusbtmeg207i07iaac.gif new file mode 100644 index 0000000..00c8312 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmusbtmeg207i07iaac.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuxvjxoj205i054dfq.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuxvjxoj205i054dfq.jpg new file mode 100644 index 0000000..0c638dd Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmuxvjxoj205i054dfq.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmv30ixuj20u00miaax.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmv30ixuj20u00miaax.jpg new file mode 100644 index 0000000..b5aca13 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmv30ixuj20u00miaax.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmvchm4oj208c08cglj.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmvchm4oj208c08cglj.jpg new file mode 100644 index 0000000..639f92f Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmvchm4oj208c08cglj.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzaqwslj2046046gln.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzaqwslj2046046gln.jpg new file mode 100644 index 0000000..4ef17ef Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzaqwslj2046046gln.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzi1v92g208c08cwf0.gif b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzi1v92g208c08cwf0.gif new file mode 100644 index 0000000..58c5f23 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzi1v92g208c08cwf0.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzsrcp5j20hs0hsn8i.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzsrcp5j20hs0hsn8i.jpg new file mode 100644 index 0000000..dd8d0b6 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugmzsrcp5j20hs0hsn8i.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugn04v6twg206o06ojts.gif b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugn04v6twg206o06ojts.gif new file mode 100644 index 0000000..bbc1665 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5gy1fugn04v6twg206o06ojts.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmsqhikzj204803oa9v.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmsqhikzj204803oa9v.jpg new file mode 100644 index 0000000..b25018d Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmsqhikzj204803oa9v.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt7qewuj201k0200sh.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt7qewuj201k0200sh.jpg new file mode 100644 index 0000000..5a35255 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt7qewuj201k0200sh.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt98nj7j206o06odfu.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt98nj7j206o06odfu.jpg new file mode 100644 index 0000000..37ad432 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmt98nj7j206o06odfu.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtc3xw6j2033039a9u.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtc3xw6j2033039a9u.jpg new file mode 100644 index 0000000..4ff8452 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtc3xw6j2033039a9u.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtdljz5j208m08swej.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtdljz5j208m08swej.jpg new file mode 100644 index 0000000..aa46e84 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtdljz5j208m08swej.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtgih1mj205i05cweb.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtgih1mj205i05cweb.jpg new file mode 100644 index 0000000..63e0208 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtgih1mj205i05cweb.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmthx0nfj204503vglg.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmthx0nfj204503vglg.jpg new file mode 100644 index 0000000..9a2344e Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmthx0nfj204503vglg.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtlivbrj206o06owfq.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtlivbrj206o06owfq.jpg new file mode 100644 index 0000000..ce318f9 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtlivbrj206o06owfq.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtn1meuj205i05iq30.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtn1meuj205i05iq30.jpg new file mode 100644 index 0000000..93a791a Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtn1meuj205i05iq30.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtq7qwgj205i03r0sl.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtq7qwgj205i03r0sl.jpg new file mode 100644 index 0000000..7137fbd Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtq7qwgj205i03r0sl.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtwtukjj204q04g0su.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtwtukjj204q04g0su.jpg new file mode 100644 index 0000000..fc8ccfb Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmtwtukjj204q04g0su.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu0ju29g208c08c3yv.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu0ju29g208c08c3yv.gif new file mode 100644 index 0000000..2965752 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu0ju29g208c08c3yv.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu1ym41j204o04ljr9.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu1ym41j204o04ljr9.jpg new file mode 100644 index 0000000..44bced5 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu1ym41j204o04ljr9.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu51rrpj20b40b4dg4.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu51rrpj20b40b4dg4.jpg new file mode 100644 index 0000000..a2565a8 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmu51rrpj20b40b4dg4.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmubsrwqj20g40g43yz.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmubsrwqj20g40g43yz.jpg new file mode 100644 index 0000000..8c3425a Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmubsrwqj20g40g43yz.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmufgtvfj206o06o0so.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmufgtvfj206o06o0so.jpg new file mode 100644 index 0000000..e280db8 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmufgtvfj206o06o0so.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuheiknj20f00qot9o.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuheiknj20f00qot9o.jpg new file mode 100644 index 0000000..5bcb028 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuheiknj20f00qot9o.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuklxcpg207i07i74f.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuklxcpg207i07i74f.gif new file mode 100644 index 0000000..998d1ca Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuklxcpg207i07i74f.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmum15i5j205i03r743.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmum15i5j205i03r743.jpg new file mode 100644 index 0000000..6e6edc8 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmum15i5j205i03r743.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmup6i0kj203o032a9w.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmup6i0kj203o032a9w.jpg new file mode 100644 index 0000000..6d8272f Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmup6i0kj203o032a9w.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuqn8aaj204q04k0sm.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuqn8aaj204q04k0sm.jpg new file mode 100644 index 0000000..6a20f4c Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuqn8aaj204q04k0sm.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmutsh2fj205a05amx3.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmutsh2fj205a05amx3.jpg new file mode 100644 index 0000000..6d4cf7b Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmutsh2fj205a05amx3.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuwdnhuj20xc0xcjtm.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuwdnhuj20xc0xcjtm.jpg new file mode 100644 index 0000000..c7d6641 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuwdnhuj20xc0xcjtm.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuzbfv7j205k05kwee.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuzbfv7j205k05kwee.jpg new file mode 100644 index 0000000..d15b0c6 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmuzbfv7j205k05kwee.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv12c5qj208c08c0t6.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv12c5qj208c08c0t6.jpg new file mode 100644 index 0000000..3f674e7 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv12c5qj208c08c0t6.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv4hwl8j205404bmx0.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv4hwl8j205404bmx0.jpg new file mode 100644 index 0000000..ebbbac9 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv4hwl8j205404bmx0.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv5y7j9j203k02y749.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv5y7j9j203k02y749.jpg new file mode 100644 index 0000000..13a7a63 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmv5y7j9j203k02y749.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz6x11sj208c08cmxm.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz6x11sj208c08cmxm.jpg new file mode 100644 index 0000000..779b96c Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz6x11sj208c08cmxm.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz8llxvg208c08c3yq.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz8llxvg208c08c3yq.gif new file mode 100644 index 0000000..b1873d4 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmz8llxvg208c08c3yq.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzehkkfj20hs0hi76b.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzehkkfj20hs0hi76b.jpg new file mode 100644 index 0000000..a1ee600 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzehkkfj20hs0hi76b.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzg8pk5j204403uaah.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzg8pk5j204403uaah.jpg new file mode 100644 index 0000000..ea329d3 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzg8pk5j204403uaah.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzjrz2ig205i05dmxj.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzjrz2ig205i05dmxj.gif new file mode 100644 index 0000000..d412ade Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzjrz2ig205i05dmxj.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzlvdh8g209p09p3zp.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzlvdh8g209p09p3zp.gif new file mode 100644 index 0000000..eb46b28 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugmzlvdh8g209p09p3zp.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn00ei38g20dc0dc40s.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn00ei38g20dc0dc40s.gif new file mode 100644 index 0000000..0228932 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn00ei38g20dc0dc40s.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn020dz3g208c08c0st.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn020dz3g208c08c0st.gif new file mode 100644 index 0000000..233dad5 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn020dz3g208c08c0st.gif differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn06a3p7j203w042jr7.jpg b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn06a3p7j203w042jr7.jpg new file mode 100644 index 0000000..bfa07bc Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn06a3p7j203w042jr7.jpg differ diff --git a/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn0dhvpng206o06o7hr.gif b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn0dhvpng206o06o7hr.gif new file mode 100644 index 0000000..6d71a94 Binary files /dev/null and b/doutulai/multithreading_spider/imagedir/9150e4e5ly1fugn0dhvpng206o06o7hr.gif differ diff --git a/doutulai/multithreading_spider/requirements.txt b/doutulai/multithreading_spider/requirements.txt new file mode 100644 index 0000000..7fabcff --- /dev/null +++ b/doutulai/multithreading_spider/requirements.txt @@ -0,0 +1,18 @@ +beautifulsoup4==4.6.3 +bs4==0.0.1 +certifi==2017.7.27.1 +cffi==1.11.5 +chardet==3.0.4 +Cheetah==2.4.4 +requests==2.18.4 +Scrapy==1.5.1 +service-identity==17.0.0 +six==1.11.0 +slip==0.4.0 +slip.dbus==0.4.0 +Twisted==18.7.0 +urlgrabber==3.10 +urllib3==1.22 +w3lib==1.19.0 +yum-metadata-parser==1.1.4 +zope.interface==4.5.0 diff --git a/doutulai/scrapy_doutulai/scrapy.cfg b/doutulai/scrapy_doutulai/scrapy.cfg new file mode 100644 index 0000000..5f219c5 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = scrapy_doutulai.settings + +[deploy] +#url = http://localhost:6800/ +project = scrapy_doutulai diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/__init__.py b/doutulai/scrapy_doutulai/scrapy_doutulai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/items.py b/doutulai/scrapy_doutulai/scrapy_doutulai/items.py new file mode 100644 index 0000000..9a24a48 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/items.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class ScrapyDoutulaiItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + # 定义图片url和name + img_url = scrapy.Field() + img_name = scrapy.Field() + diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/main.py b/doutulai/scrapy_doutulai/scrapy_doutulai/main.py new file mode 100644 index 0000000..7e82bf1 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/main.py @@ -0,0 +1,8 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +from scrapy import cmdline +# 执行函数 +cmdline.execute('scrapy crawl doutulai_spider'.split()) + diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py b/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py new file mode 100644 index 0000000..22d240e --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class ScrapyDoutulaiSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class ScrapyDoutulaiDownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py b/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py new file mode 100644 index 0000000..98cfd04 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +import os +from urllib import urlretrieve +from scrapy_doutulai.settings import DOWNLOAD_DIR + +class ScrapyDoutulaiPipeline(object): + def __init__(self): + """ + 判断下载目录是否存在 + """ + if not os.path.exists(DOWNLOAD_DIR): + os.makedirs(DOWNLOAD_DIR) + + def process_item(self, item, spider): + """ + 下载图片 + :param item: + :param spider: + :return: + """ + try: + filename = os.path.join(DOWNLOAD_DIR,item['img_name']) + print(filename) + urlretrieve(item['img_url'],filename) + except Exception as e: + pass \ No newline at end of file diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py b/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py new file mode 100644 index 0000000..215e3fc --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for scrapy_doutulai project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'scrapy_doutulai' + +SPIDER_MODULES = ['scrapy_doutulai.spiders'] +NEWSPIDER_MODULE = 'scrapy_doutulai.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'scrapy_doutulai (+http://www.yourdomain.com)' +USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'scrapy_doutulai.middlewares.ScrapyDoutulaiSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'scrapy_doutulai.middlewares.ScrapyDoutulaiDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://doc.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'scrapy_doutulai.pipelines.ScrapyDoutulaiPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +# 定义图片下载目录 +DOWNLOAD_DIR = 'imagedir' \ No newline at end of file diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc new file mode 100644 index 0000000..1e893c8 Binary files /dev/null and b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc differ diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.py b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.py new file mode 100644 index 0000000..2131861 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +import scrapy + +from scrapy_doutulai.items import ScrapyDoutulaiItem + +class DoutulaiSpiderSpider(scrapy.Spider): + name = 'doutulai_spider' + allowed_domains = ['www.doutula.com'] + start_urls = ['https://www.doutula.com/photo/list/'] + page = 1 + + def parse(self, response): + content_items = ScrapyDoutulaiItem() + # 解析img_url列表,拿到图片的url和,图片名称 + img_url_list = response.xpath('//img[@class="img-responsive lazy image_dta"]') + # page_number = response.xpath('//*[@id="pic-detail"]/div/div[3]/div[3]/ul/li[12]/a/text()').extract_first() + page_number = response.xpath('//a[@class="page-link"][last()]/text()').extract_first() + + for img_content in img_url_list: + content_items['img_url'] = img_content.xpath('./@data-original').extract_first() + content_items['img_name'] = img_content.xpath('./@data-original').extract_first().split('/')[-1] + print(content_items) + yield content_items + # 不断爬取新页面 + if self.page <= page_number: + self.page += 1 + next_url = self.start_urls[0] + '?page=' + str(self.page) + yield scrapy.Request(next_url) \ No newline at end of file diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.pyc b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.pyc new file mode 100644 index 0000000..a713edb Binary files /dev/null and b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.pyc differ diff --git a/get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc b/get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc new file mode 100644 index 0000000..d6af67a Binary files /dev/null and b/get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc differ diff --git a/get_RdsBakupDownloadUrl/__pycache__/set_log.cpython-36.pyc b/get_RdsBakupDownloadUrl/__pycache__/set_log.cpython-36.pyc new file mode 100644 index 0000000..04a8f7e Binary files /dev/null and b/get_RdsBakupDownloadUrl/__pycache__/set_log.cpython-36.pyc differ diff --git a/get_RdsBakupDownloadUrl/get_rdsinfo.py b/get_RdsBakupDownloadUrl/get_rdsinfo.py new file mode 100644 index 0000000..db10d3f --- /dev/null +++ b/get_RdsBakupDownloadUrl/get_rdsinfo.py @@ -0,0 +1,73 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from aliyunsdkcore import client +from aliyunsdkrds.request.v20140815 import DescribeBackupsRequest +import json +import datetime +import configparser + + +class rdsOper(): + def __init__(self,logger): + """ + defined start and end time + init clentoper + """ + configoper = configparser.ConfigParser() + configoper.read('info.conf') + self.accessKeyId = configoper['akconfig']['accessKeyId'] + self.accessSecret = configoper['akconfig']['accessSecret'] + self.region = configoper['akconfig']['region'] + self.instanceId = configoper['akconfig']['instanceId'] + self.BackupMode = configoper['akconfig']['BackupMode'] + self.BackupDownURL = configoper['akconfig']['BackDownloadURL'] + self.clent_oper = client.AcsClient(self.accessKeyId, self.accessSecret, self.region) + self.start_time = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") + 'T00:00Z' + self.end_time = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") + 'T23:59Z' + self.logger = logger + + def des_instance(self): + """ + set param + :return: response + """ + request = DescribeBackupsRequest.DescribeBackupsRequest() + request.set_accept_format('json') + # DBInstanceId: + request.add_query_param('DBInstanceId', self.instanceId) + # StartTime: + request.add_query_param('StartTime', self.start_time) + # EndTime: + request.add_query_param('EndTime', self.end_time) + # BackupId: + # request.add_query_param('BackupId', '备份集ID') + # BackupStatus: + request.add_query_param('BackupStatus', 'Success') + # 备份类型,取值范围:Automated:常规任务 + request.add_query_param('BackupMode', self.BackupMode) + response = self.clent_oper.do_action_with_exception(request) + self.logger.info("init request complate!") + return response + + # 接受响应,定义外网或内网url,公网:BackupDownloadURL,内网:BackupIntranetDownloadURL + def get_rdsdownload_url(self, response, net_type='BackupDownloadURL'): + """ + :param response: + :param net_type: + :return: rds_download_url + """ + context = json.loads(response.decode('utf-8')) + con_list = context['Items']['Backup'] + net_type = self.BackupDownURL + try: + download_url = con_list[0][net_type] + + log_url = str(datetime.datetime.now()) +" downloadurl:"+ download_url + self.logger.info(log_url) + except Exception as e: + self.logger.info(e) + exit() + return download_url + diff --git a/get_RdsBakupDownloadUrl/info.conf b/get_RdsBakupDownloadUrl/info.conf new file mode 100644 index 0000000..836a261 --- /dev/null +++ b/get_RdsBakupDownloadUrl/info.conf @@ -0,0 +1,30 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch +#Get the download address of the RDS backupdownload url, if you want to get other information, you can modify the code yourself. + +[akconfig] +#Configure Ali cloud accesskeyid +accessKeyId = LTAIhfXlcjyln6tW + +#Configure Ali cloud accessscret +accessSecret = GwfAMvR4K2ELmt76184oqLTVgRfAso + +#Configuring the area where the RDS instance is located +region = cn-beijing + +#Instance ID for configuring RDS +instanceId = rm-2zer20742k7bl5b56 + +#Configure RDS backup type type:"Automated" system automatically backup, "Manual" manually perform backup +BackupMode = Manual + +Download URL type configured: type:"BackupIntranetDownloadURL" intranet download address, "BackupDownloadURL" public network download address +BackDownloadURL = BackupDownloadURL + +# loginfo configuration +[logconfig] +#Log file directory name +logdir_name = rdsbacklog +#Log file name +logfile_name = desrdslog.log diff --git a/get_RdsBakupDownloadUrl/main.py b/get_RdsBakupDownloadUrl/main.py new file mode 100644 index 0000000..0ac2169 --- /dev/null +++ b/get_RdsBakupDownloadUrl/main.py @@ -0,0 +1,31 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from get_rdsinfo import rdsOper +from set_log import rdsLog + + +def init_log(): + """ + init logger + :return: logger + """ + logoper = rdsLog() + filename = logoper.create_dir() + rdslog = logoper.create_logger(filename) + return rdslog + +def app_rdsbak(): + """ + + :return: + """ + log = init_log() + rdsapp = rdsOper(log) + result = rdsapp.des_instance() +# print(rdsapp.get_rdsdownload_url(result, 'BackupIntranetDownloadURL')) + print(rdsapp.get_rdsdownload_url(result, 'BackupDownloadURL')) + +if __name__ == "__main__": + app_rdsbak() diff --git a/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log b/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log new file mode 100644 index 0000000..0a401a6 --- /dev/null +++ b/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log @@ -0,0 +1,2 @@ +2018-07-04 13:52:41,389 - root - INFO - init request complate! +2018-07-04 13:52:41,389 - root - INFO - 2018-07-04 13:52:41.389141 downloadurl:http://rdsbak-bjc-v2.oss-cn-beijing-c.aliyuncs.com/custins7231221/hins4953173_data_20180704113854.tar.gz?OSSAccessKeyId=LTAITfQ7krsrEwRn&Expires=1530769960&Signature=1XEZhUjQFCNir6rpd4omo71NyGs%3D diff --git a/get_RdsBakupDownloadUrl/requirements.txt b/get_RdsBakupDownloadUrl/requirements.txt new file mode 100644 index 0000000..13c9dba --- /dev/null +++ b/get_RdsBakupDownloadUrl/requirements.txt @@ -0,0 +1,15 @@ +aliyun-python-sdk-core==2.3.5 +aliyun-python-sdk-core-v3==2.8.6 +aliyun-python-sdk-ecs==4.7.1 +aliyun-python-sdk-rds==2.1.3 +certifi==2018.1.18 +chardet==3.0.4 +EasyProcess==0.2.3 +idna==2.6 +ipython==5.3.0 +pycryptodome==3.6.1 +PyVirtualDisplay==0.2.1 +requests==2.18.4 +selenium==3.4.3 +urllib3==1.22 +XlsxWriter==0.9.6 diff --git a/get_RdsBakupDownloadUrl/set_log.py b/get_RdsBakupDownloadUrl/set_log.py new file mode 100644 index 0000000..f472afe --- /dev/null +++ b/get_RdsBakupDownloadUrl/set_log.py @@ -0,0 +1,33 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch +import os +import time +import logging +import configparser + +class rdsLog: + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('info.conf') + self.logdir_name = configoper['logconfig']['logdir_name'] + self.logfile_name = configoper['logconfig']['logfile_name'] + + def create_dir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger diff --git a/imageoper/__pycache__/logger.cpython-36.pyc b/imageoper/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000..db1d2b9 Binary files /dev/null and b/imageoper/__pycache__/logger.cpython-36.pyc differ diff --git a/imageoper/config.cfg b/imageoper/config.cfg new file mode 100644 index 0000000..56828a6 --- /dev/null +++ b/imageoper/config.cfg @@ -0,0 +1,38 @@ +# 阿里云ak配置,建议采用子账户只授权ecs镜像操作 +[common] +# 阿里云acccesskeyid +accessKeyId = LTAIhfxxxxxxxx +# 阿里云accesssecret +accessSecret = GwfAMvR4Kxxxxxxxxxxxxxxxxxxx +# log目录名称 +logdir_name = logdir +# log文件名称 +logfile_name = ecsoperlog.log + +# ecs源地域配置信息段 +#支持在华北 1、华北 2、华北 3、华北 5、华东 1、华东 2 和华南 1 地域之间复制镜像。涉及其他国家和地区地域时,可以 提交工单 申请 +[source] +# 源地域实例regionid,可以参考:https://help.aliyun.com/document_detail/40654.html?spm=a2c1g.8271268.10000.5.5f98df25B98bhJ +s_RegionId = cn-shanghai + +# 源实例id,可指定多个用,进行分隔 +s_InstanceId = i-uf661wb708uvqc9jyhem,i-uf661wb708uvqc9jyhel + +# 源端制作镜像name +s_ImageName = api-source-image + +# 源镜像描述信息 +s_Description = api-source-image源镜像描述信息 + +# 镜像复制目的地域配置信息段 +[destination] +# 目的地域实例regionid, +d_DestinationRegionId = cn-qingdao + +# 复制过来的镜像名称 +d_DestinationImageName = api-destination-image + +# 复制过来的镜像描述信息 +d_DestinationDescription = api-destination-image目的镜像描述信息 + + diff --git a/imageoper/create_cop_image.py b/imageoper/create_cop_image.py new file mode 100644 index 0000000..9a948c3 --- /dev/null +++ b/imageoper/create_cop_image.py @@ -0,0 +1,122 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import configparser +import time +import json +from aliyunsdkcore import client +from aliyunsdkecs.request.v20140526 import CreateImageRequest +from aliyunsdkecs.request.v20140526 import CopyImageRequest +from aliyunsdkecs.request.v20140526 import DescribeImagesRequest +import logger + + +class EcsOper(): + def __init__(self,logger): + """ + 初始化获取系config信息,配置日志 + :param logger: + """ + config = configparser.ConfigParser() + config.read('config.cfg',encoding='utf-8') + accessKeyId = config['common']['accessKeyId'] + accessSecret = config['common']['accessSecret'] + self.s_RegionId = config['source']['s_RegionId'] + self.s_InstanceId_list = config['source']['s_InstanceId'] + self.s_ImageName = config['source']['s_ImageName'] + self.s_Description = config['source']['s_Description'] + + self.d_DestinationRegionId = config['destination']['d_DestinationRegionId'] + self.d_DestinationImageName = config['destination']['d_DestinationImageName'] + self.d_DestinationDescription = config['destination']['d_DestinationDescription'] + self.ecshelper = client.AcsClient(accessKeyId,accessSecret,self.s_RegionId) + + logger = logger.LogHelper() + logname = logger.create_dir() + self.logoper = logger.create_logger(logname) + + # 创建实例生成器 + def _get_Instance(self): + for Instance in self.s_InstanceId_list.split(','): + yield Instance + + # 镜像制作 + def _create_image(self,s_InstanceId): + """ + 创建镜像 + :return:返回镜像id + """ + s_timer = time.strftime("%Y-%m-%d-%H:%M", time.localtime(time.time())) + request = CreateImageRequest.CreateImageRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('InstanceId', s_InstanceId) + request.add_query_param('ImageName', self.s_ImageName + s_timer) + request.add_query_param('Description', self.s_Description + s_timer) + response = self.ecshelper.do_action_with_exception(request) + self.logoper.info('创建镜像任务已提交,镜像id:%s' % json.loads(response)["ImageId"]) + print('实例%s,创建镜像任务已提交,镜像id:%s' % (s_InstanceId,json.loads(response)["ImageId"])) + return json.loads(response)["ImageId"] + + # 查询镜像状态 + def _describe_image(self,imageid): + """ + 查询image状态 + :param imageid: + :return: + """ + request = DescribeImagesRequest.DescribeImagesRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('ImageId', imageid) + response = self.ecshelper.do_action_with_exception(request) + # 进度 json.loads(response)['Images']['Image'][0]['Progress'] + self.logoper.info('镜像创建进度:%s' %json.loads(response)['Images']['Image'][0]['Progress']) + # 镜像状态 + return json.loads(response)['Images']['Image'][0]['Status'] + + #镜像复制 + def _copy_image(self,imageid): + """ + 镜像复制 + :param imageid:源镜像id + :return: 复制成功后的镜像id + """ + flag = True + while flag: + try: + if self._describe_image(imageid) == 'Available': + flag = False + else: + time.sleep(300) + except Exception as e: + pass + print('镜像已经创建完成') + d_timer = time.strftime("%Y-%m-%d-%H:%M", time.localtime(time.time())) + request = CopyImageRequest.CopyImageRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('DestinationRegionId', self.d_DestinationRegionId) + request.add_query_param('DestinationImageName', self.d_DestinationImageName + d_timer) + request.add_query_param('DestinationDescription', self.d_DestinationDescription + d_timer) + request.add_query_param('ImageId', imageid) + response = self.ecshelper.do_action_with_exception(request) + self.logoper.info('复制镜像任务已提交,镜像id:%s' % json.loads(response)['ImageId']) + print('复制镜像任务已提交,镜像id:%s' % json.loads(response)['ImageId']) + return json.loads(response)['ImageId'] + + def run(self): + for instance_id in self._get_Instance(): + s_imageid = self._create_image(instance_id) + self._copy_image(s_imageid) + +if __name__ == '__main__': + ecsoper = EcsOper(logger) + ecsoper.run() + + + + + + diff --git a/imageoper/logdir/2018-07-25-ecsoperlog.log b/imageoper/logdir/2018-07-25-ecsoperlog.log new file mode 100644 index 0000000..f2c3e93 --- /dev/null +++ b/imageoper/logdir/2018-07-25-ecsoperlog.log @@ -0,0 +1,3 @@ +2018-07-25 16:13:07,098 - root - INFO - ύ,id:m-uf6eby4dmp9llm9dgz50 +2018-07-25 16:18:09,721 - root - INFO - 񴴽:100% +2018-07-25 16:18:10,067 - root - INFO - ƾύ,id:m-m5eb53zrph7pic6zuejq diff --git a/imageoper/logdir/2018-07-26-ecsoperlog.log b/imageoper/logdir/2018-07-26-ecsoperlog.log new file mode 100644 index 0000000..9fe16ff --- /dev/null +++ b/imageoper/logdir/2018-07-26-ecsoperlog.log @@ -0,0 +1,6 @@ +2018-07-26 10:16:59,150 - root - INFO - ύ,id:m-uf6cp52t42s72i4jgxli +2018-07-26 10:22:00,824 - root - INFO - 񴴽:100% +2018-07-26 10:22:01,314 - root - INFO - ƾύ,id:m-m5e8b39oaqnhbfp9x35i +2018-07-26 10:22:03,383 - root - INFO - ύ,id:m-uf60h59flke4cqhtg6nn +2018-07-26 10:27:04,725 - root - INFO - 񴴽:100% +2018-07-26 10:27:05,060 - root - INFO - ƾύ,id:m-m5ednmzrres9mkjjsq0l diff --git a/imageoper/logger.py b/imageoper/logger.py new file mode 100644 index 0000000..338d1dd --- /dev/null +++ b/imageoper/logger.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + +import os +import time +import logging +import configparser + +class LogHelper(): + """ + 初始化logger,读取目录及文件名称 + """ + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('config.cfg',encoding='utf-8') + self.logdir_name = configoper['common']['logdir_name'] + self.logfile_name = configoper['common']['logfile_name'] + + def create_dir(self): + """ + 创建目录 + :return: 文件名称 + """ + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self, logfilename): + """ + 创建logger对象 + :param logfilename: + :return: logger对象 + """ + logger = logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6bd4f36 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +aliyun-python-sdk-core==2.3.5 +aliyun-python-sdk-core-v3==2.8.6 +aliyun-python-sdk-ecs==4.7.1 +aliyun-python-sdk-rds==2.1.3 +certifi==2018.1.18 +chardet==3.0.4 +EasyProcess==0.2.3 +idna==2.6 +ipython==5.3.0 +pycryptodome==3.6.1 +PyVirtualDisplay==0.2.1 +requests==2.18.4 +selenium==3.4.3 +urllib3==1.22 +XlsxWriter==0.9.6 +tencentcloud-sdk-python==3.0.20 + diff --git a/save_blog/save_blog.py b/save_blog/save_blog.py new file mode 100644 index 0000000..3290252 --- /dev/null +++ b/save_blog/save_blog.py @@ -0,0 +1,101 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +import time +from bs4 import BeautifulSoup +from selenium import webdriver + + +class BlogSave(): + # 定义headers字段 + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36" + } + + def __init__(self,blog_name,page_number,login_user_name,login_passwd): + self.login_url = 'http://home.51cto.com/index' + # 博客用户名 + self.blog_name = blog_name + # 需要保存的博客多少页 + self.page_number = page_number + # 登陆的用户 + self.login_user_name = login_user_name + # 登陆的密码 + self.login_passwd = login_passwd + # 本地的chreomedriver驱动 + self.chromedirve = 'D:\chromedriver.exe' + # blog 导入url + self.blog_save_url = 'http://blog.51cto.com/blogger/publish/' + + + def get_urldict(self): + """ + 爬去用户文章的url + :param pagenumber: + :return: urllist + """ + content_dict = {} + scrapy_urllist = ["http://blog.51cto.com/" + str(self.blog_name) + "/p" + str(page) for page in + range(1, int(self.page_number) + 1)] + for scrapy_url in scrapy_urllist: + response = requests.get(scrapy_url, headers=BlogSave.headers) + soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf-8') + title_list = soup.find_all('a', class_='tit') + + for content in title_list: + # 获取url + url = content['href'] + title_soup = BeautifulSoup(requests.get(url, headers=BlogSave.headers).content, 'lxml', from_encoding='utf-8') + title = title_soup.find_all('h1', class_='artical-title') + # 获取标题 + # print(title[0].get_text()) + content_dict[title[0].get_text()] = url + print(title[0].get_text(),url) + + return content_dict + + + def save_blog(self,url_list): + """ + 通过模拟登陆保存博客文件 + :return: + """ + browser = webdriver.Chrome(self.chromedirve) + # 打开url + browser.get(self.login_url) + time.sleep(2) + # 登陆 + browser.find_element_by_id('loginform-username').send_keys(self.login_user_name) + browser.find_element_by_id('loginform-password').send_keys(self.login_passwd) + browser.find_element_by_name('login-button').click() + time.sleep(1) + for url in url_list: + browser.get(url) + time.sleep(1) + try: + browser.find_element_by_xpath('//*[@id="blogEditor-box"]/div[1]/a[14]').click() + time.sleep(2) + except Exception as e: + with open('fail.log','a') as f: + f.write(url + str(e)) + + def run(self): + # 获取标题和url字典 + content_dict = self.get_urldict() + # 获取url列表 + id_list = [] + for value in content_dict.values(): + id_list.append(str(value).split('/')[-1]) + result_list = [ self.blog_save_url + str(id) for id in id_list ] + print("result_list:",result_list) + self.save_blog(result_list) + +if __name__ == '__main__': + # blogOper = BlogSave('kaliarch',1) + # dict = blogOper.get_urldict() + # value_list = [ value for value in dict.values()] + # print(value_list) + blogOper = BlogSave(blog_name='kaliarch',page_number=5,login_user_name='xxxxxxxxxxxxx@163.com',login_passwd='qxxxxxxxxx') + blogOper.run() diff --git a/tcp_port_check/__init__.py b/tcp_port_check/__init__.py new file mode 100644 index 0000000..a4af0aa --- /dev/null +++ b/tcp_port_check/__init__.py @@ -0,0 +1,5 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + diff --git a/tcp_port_check/__pycache__/logger.cpython-36.pyc b/tcp_port_check/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000..b1ddc3e Binary files /dev/null and b/tcp_port_check/__pycache__/logger.cpython-36.pyc differ diff --git a/tcp_port_check/info.cfg b/tcp_port_check/info.cfg new file mode 100644 index 0000000..631d5ad --- /dev/null +++ b/tcp_port_check/info.cfg @@ -0,0 +1,19 @@ +#公网端口检测配置 +[port_check_info] + +#检测ip地址或域名 +#address = baidu.com +#address = 8.8.8.8 +address = www.51cto.com,www.anchnet.com + +#检查的端口,如多个端口使用,隔开,端口范围使用'-' +#ports = 80,8080.... +ports = 20-25,80,443,1433,1521,3306,3389,6379,8080,27017 + +#日志配置 +[loginfo] +#日志目录 +logdir_name = logdir + +#日志文件名称 +logfile_name = check_port.log \ No newline at end of file diff --git a/tcp_port_check/logdir/2018-07-23-check_port.log b/tcp_port_check/logdir/2018-07-23-check_port.log new file mode 100644 index 0000000..23303c1 --- /dev/null +++ b/tcp_port_check/logdir/2018-07-23-check_port.log @@ -0,0 +1,30 @@ +2018-07-23 23:07:55,506 - root - INFO - www.51cto.com:20,port status is:['ر'] +2018-07-23 23:07:56,985 - root - INFO - www.51cto.com:21,port status is:['ر'] +2018-07-23 23:07:58,705 - root - INFO - www.51cto.com:22,port status is:['ر'] +2018-07-23 23:08:00,786 - root - INFO - www.51cto.com:23,port status is:['ر'] +2018-07-23 23:08:02,866 - root - INFO - www.51cto.com:24,port status is:['ر'] +2018-07-23 23:08:04,383 - root - INFO - www.51cto.com:25,port status is:['ر'] +2018-07-23 23:08:09,746 - root - INFO - www.anchnet.com:20,port status is:['ر'] +2018-07-23 23:08:15,187 - root - INFO - www.anchnet.com:21,port status is:['ر'] +2018-07-23 23:08:20,626 - root - INFO - www.anchnet.com:22,port status is:['ر'] +2018-07-23 23:08:25,917 - root - INFO - www.anchnet.com:23,port status is:['ر'] +2018-07-23 23:08:31,506 - root - INFO - www.anchnet.com:24,port status is:['ر'] +2018-07-23 23:08:37,585 - root - INFO - www.anchnet.com:25,port status is:['ر'] +2018-07-23 23:08:38,385 - root - INFO - www.51cto.com:80,port status is:[''] +2018-07-23 23:08:38,903 - root - INFO - www.51cto.com:443,port status is:[''] +2018-07-23 23:08:40,946 - root - INFO - www.51cto.com:1433,port status is:['ر'] +2018-07-23 23:08:42,705 - root - INFO - www.51cto.com:1521,port status is:['ر'] +2018-07-23 23:08:45,002 - root - INFO - www.51cto.com:3306,port status is:['ر'] +2018-07-23 23:08:47,026 - root - INFO - www.51cto.com:3389,port status is:['ر'] +2018-07-23 23:08:48,627 - root - INFO - www.51cto.com:6379,port status is:['ر'] +2018-07-23 23:08:49,291 - root - INFO - www.51cto.com:8080,port status is:[''] +2018-07-23 23:08:50,788 - root - INFO - www.51cto.com:27017,port status is:['ر'] +2018-07-23 23:08:51,668 - root - INFO - www.anchnet.com:80,port status is:[''] +2018-07-23 23:08:52,751 - root - INFO - www.anchnet.com:443,port status is:[''] +2018-07-23 23:08:58,148 - root - INFO - www.anchnet.com:1433,port status is:['ر'] +2018-07-23 23:09:05,105 - root - INFO - www.anchnet.com:1521,port status is:['ر'] +2018-07-23 23:09:10,555 - root - INFO - www.anchnet.com:3306,port status is:['ر'] +2018-07-23 23:09:16,032 - root - INFO - www.anchnet.com:3389,port status is:['ر'] +2018-07-23 23:09:22,466 - root - INFO - www.anchnet.com:6379,port status is:['ر'] +2018-07-23 23:09:28,314 - root - INFO - www.anchnet.com:8080,port status is:['ر'] +2018-07-23 23:09:34,545 - root - INFO - www.anchnet.com:27017,port status is:['ر'] diff --git a/tcp_port_check/logger.py b/tcp_port_check/logger.py new file mode 100644 index 0000000..c2e4095 --- /dev/null +++ b/tcp_port_check/logger.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + +import os +import time +import logging +import configparser + +class LogHelper(): + """ + 初始化logger,读取目录及文件名称 + """ + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('info.cfg',encoding='utf-8') + self.logdir_name = configoper['loginfo']['logdir_name'] + self.logfile_name = configoper['loginfo']['logfile_name'] + + def create_dir(self): + """ + 创建目录 + :return: 文件名称 + """ + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self, logfilename): + """ + 创建logger对象 + :param logfilename: + :return: logger对象 + """ + logger = logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger diff --git a/tcp_port_check/tcp_port_check.py b/tcp_port_check/tcp_port_check.py new file mode 100644 index 0000000..1ee732c --- /dev/null +++ b/tcp_port_check/tcp_port_check.py @@ -0,0 +1,95 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +from configparser import ConfigParser +import re +import logger + +class check_ports(): + def __init__(self,logger): + """ + 初始化,获取配置文件信息 + """ + self.url = 'http://tool.chinaz.com/iframe.ashx?t=port' + self.headers = { + 'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.8', + 'Connection': 'keep-alive', + 'Content-Length': '62', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Host': 'tool.chinaz.com', + 'Origin': 'http://tool.chinaz.com', + 'Referer': 'http://tool.chinaz.com/port/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36', + 'X-Requested-With': 'XMLHttpRequest' + } + config = ConfigParser() + config.read('info.cfg',encoding='utf-8') + self.address_list = config['port_check_info']['address'] + self.port_list = config['port_check_info']['ports'] + #初始化logger + logger = logger.LogHelper() + logname = logger.create_dir() + self.logoper = logger.create_logger(logname) + + + def _get_body(self): + """ + 获取address和port + :return: list + """ + address_list = self.address_list.split(',') + port_list = self.port_list.split(',') + + # 处理端口范围,返回range + range_flag = False + port_range = None + content_List_range = [] + for port in port_list: + if '-' in port: + range_flag = True + port_range = range(int(port.split('-')[0]),int(port.split('-')[1])+1) + port_list.remove(port) + + # 处理总体list + for add in address_list: + if range_flag: + for port in port_range: + content_List_range.append(add + ':' + str(port)) + + # 合并range和普通list + content_List = [ add+':'+port for add in address_list for port in port_list ] + content_List_range.extend(content_List) + return content_List_range + + def run(self): + """ + 进行端口检测 + :return: + """ + for content in self._get_body(): + content_list = content.split(':') + body = { + 'host': content_list[0], + 'port': content_list[1], + 'encode': 'tlCHS1u3IgF4sC57m6KOP3Oaj1Y1kfLq' + } + try: + response = requests.post(url=self.url,data=body,headers=self.headers) + port_status = re.findall("msg:'(.*?)'", response.text) + if len(port_status) > 0: + print('%s,port status is:%s' % (content, port_status)) + self.logoper.info('%s,port status is:%s' % (content, port_status)) + else: + self.logoper.info('%s,port status is:%s' % (content, port_status)) + print('Occer error!请输入正确的地址和端口') + except Exception as e: + self.logoper.info(e) + + +if __name__ == '__main__': + check_app = check_ports(logger) + check_app.run() \ No newline at end of file