From fdf4a96f7bb4bd2aec7e5dc5e6d1d12cc9abe616 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Wed, 7 Mar 2018 16:17:32 +0800 Subject: [PATCH 01/21] change README and add file --- PROXIES/db.conf | 31 +++++++++++++++++++ PROXIES/save_memcache.py | 47 +++++++++++++++++++++++++++++ PROXIES/save_mongodb.py | 59 ++++++++++++++++++++++++++++++++++++ PROXIES/save_mysql.py | 64 ++++++++++++++++++++++++++++++++++++++++ PROXIES/save_redis.py | 52 ++++++++++++++++++++++++++++++++ PROXIES/spider.py | 61 ++++++++++++++++++++++++++++++++++++++ README.md | 20 +++++++++++++ 7 files changed, 334 insertions(+) create mode 100644 PROXIES/db.conf create mode 100644 PROXIES/save_memcache.py create mode 100644 PROXIES/save_mongodb.py create mode 100644 PROXIES/save_mysql.py create mode 100644 PROXIES/save_redis.py create mode 100644 PROXIES/spider.py diff --git a/PROXIES/db.conf b/PROXIES/db.conf new file mode 100644 index 0000000..1066d95 --- /dev/null +++ b/PROXIES/db.conf @@ -0,0 +1,31 @@ +[mysql] + +HOST = 172.20.6.100 +PORT = 3306 +USER = root +PASSWD = mysqladmin +DB = pydb +TABLE = pytab +CHARSET = utf8 + + +[redis] + +HOST = 172.20.6.100 +PORT = 6379 +PASSWD = redisadmin + + +[memcache] + +HOST = 172.20.6.100 +PORT = 11211 + + +[mongodb] + +HOST = 172.20.6.100 +PORT = 27017 +DB = db1 +USER = mongoadmin +PASSWD = mongopwd diff --git a/PROXIES/save_memcache.py b/PROXIES/save_memcache.py new file mode 100644 index 0000000..8edf889 --- /dev/null +++ b/PROXIES/save_memcache.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import memcache +import random +import configparser +import spider + +class MemcacheOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['memcache']['HOST'] + self.port = config['memcache']['PORT'] + self.mcoper = memcache.Client([self.host+':'+self.port], debug = True) + + def memcache_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.mcoper.set(str(num),cont) + + def memcache_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.mcoper.get(str(num)) + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MemcacheOper() + dbhelper.memcache_save(proxy_ip) + ip = dbhelper.memcache_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mongodb.py b/PROXIES/save_mongodb.py new file mode 100644 index 0000000..0f76994 --- /dev/null +++ b/PROXIES/save_mongodb.py @@ -0,0 +1,59 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + + + +import configparser +import spider +from pymongo import MongoClient + +class MongodbOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mongodb']['HOST'] + self.port = config['mongodb']['PORT'] + self.db = config['mongodb']['DB'] + self.user = config['mongodb']['USER'] + self.pwd = config['mongodb']['PASSWD'] + self.client = MongoClient(self.host, int(self.port)) + self.db_auth = self.client.admin + self.db_auth.authenticate(self.user,self.pwd) + self.DB = self.client[self.db] + self.collection = self.DB.myset + + def mongodb_save(self,result_list): + """ + save data + :return:None + """ + + for values in result_list: + self.collection.insert(values) + + def mongodb_gain(self): + """ + gain data + :return: proxies + """ + ip = self.collection.find_one() + return ip + +if __name__ == '__main__': + proxyhelper = spider.GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MongodbOper() + dbhelper.mongodb_save(proxy_ip) + ip = dbhelper.mongodb_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/save_mysql.py b/PROXIES/save_mysql.py new file mode 100644 index 0000000..f7c40c7 --- /dev/null +++ b/PROXIES/save_mysql.py @@ -0,0 +1,64 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import pymysql +import configparser +import spider + + +class MysqlOper: + # initial database information + def __init__(self, result_list): + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['mysql']['HOST'] + self.port = int(config['mysql']['PORT']) + self.user = config['mysql']['USER'] + self.passwd = config['mysql']['PASSWD'] + self.db = config['mysql']['DB'] + self.table = config['mysql']['TABLE'] + self.charset = config['mysql']['CHARSET'] + self.result_list = result_list + + def mysql_save(self): + + # create db cursor + try: + DB = pymysql.connect(self.host, self.user, self.passwd, self.db, port=self.port, charset=self.charset) + cursor = DB.cursor() + except Exception as e: + print("connect dbserver fail,Please see information:") + print(e) + exit(1) + + # check and create tables + cursor.execute('show tables in pydb') + tables = cursor.fetchall() + flag = True + for tab in tables: + if self.table in tab: + flag = False + print('%s is exist' % self.table) + print(flag) + if flag: + cursor.execute( + '''create table pytab (id int unsigned not null primary key auto_increment, protocol varchar(10),content varchar(50))''') + else: + return 0 + + # write database + for values in self.result_list: + for prot, cont in values.items(): + try: + cursor.execute("insert into pytab (protocol,content) value (%s,%s);", [prot, cont]) + except Exception as e: + print("insert db occer error", e) + + +if __name__ == "__main__": + proxyhelper = spider.GetProxyIP(3) + res_pool = proxyhelper.get_ip() + proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = MysqlOper(proxy_ip) + dbhelper.mysql_save() diff --git a/PROXIES/save_redis.py b/PROXIES/save_redis.py new file mode 100644 index 0000000..5c7bb91 --- /dev/null +++ b/PROXIES/save_redis.py @@ -0,0 +1,52 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import redis +import random +import configparser +import spider + +class RedisOper: + + def __init__(self): + """ + initialization redis infomation + :param + """ + config = configparser.ConfigParser() + config.read('db.conf') + self.host = config['redis']['HOST'] + self.port = config['redis']['PORT'] + self.passwd = config['redis']['PASSWD'] + self.pool = redis.ConnectionPool(host=self.host,port=self.port,password=self.passwd) + self.redis_helper = redis.Redis(connection_pool=self.pool) + self.pipe = self.redis_helper.pipeline(transaction=True) + + def redis_save(self,result_list): + """ + save data + :return:None + """ + for num,cont in enumerate(result_list): + self.redis_helper.set(num,cont) + self.pipe.execute() + + def redis_gain(self): + """ + gain data + :return: proxies + """ + num = random.randint(0,10) + ip = self.redis_helper.get(num) + self.pipe.execute() + return ip + +if __name__ == '__main__': + # proxyhelper = spider.GetProxyIP(2) + # res_pool = proxyhelper.get_ip() + # proxy_ip = proxyhelper.right_proxies(res_pool) + dbhelper = RedisOper() + # dbhelper.redis_save(proxy_ip) + ip = dbhelper.redis_gain() + print(ip) \ No newline at end of file diff --git a/PROXIES/spider.py b/PROXIES/spider.py new file mode 100644 index 0000000..9b538c6 --- /dev/null +++ b/PROXIES/spider.py @@ -0,0 +1,61 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +import requests +from bs4 import BeautifulSoup +import random + +class GetProxyIP: + + def __init__(self,page=10): + self._page = page + self.url_head = 'http://www.xicidaili.com/wt/' + + def get_ip(self): + """ + get resouce proxy ip pool + :return: res_pool list + """ + headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"} + res_pool = [] + for pagenum in range(1,self._page): + url = self.url_head + str(pagenum) + response = requests.get(url, headers=headers) + soup = BeautifulSoup(response.text, "html.parser") + soup_tr = soup.find_all('tr') + for item in soup_tr: + try: + soup_td = item.find_all('td') + res_pool.append(soup_td[5].text.lower() + '://' + soup_td[1].text + ':' + soup_td[2].text) + except IndexError: + pass + return res_pool + + def right_proxies(self,res_pool): + """ + check available ip + :param res_pool: + :return:right_pool list + """ + right_pool = [] + for ip in res_pool: + if 'https' in ip: + proxies = {'http': ip} + else: + proxies = {"http": ip} + check_urllist = ['http://www.baidu.com', 'http://www.taobao.com', 'https://cloud.tencent.com/'] + try: + response = requests.get(random.choice(check_urllist), proxies=proxies, timeout = 1) + if response.status_code: + right_pool.append(proxies) + print('add ip %s' % proxies) + except Exception as e: + continue + return right_pool + +if __name__ == '__main__': + proxyhelper = GetProxyIP(2) + res_pool = proxyhelper.get_ip() + proxy_ip =proxyhelper.right_proxies(res_pool) + print(proxy_ip) diff --git a/README.md b/README.md index 082ea2b..de6a825 100644 --- a/README.md +++ b/README.md @@ -1 +1,21 @@ # my-python-code +## 翻译小工具 +> Python实现翻译小工具 + +blog：[Python实现翻译小工具](http://blog.51cto.com/kaliarch/2072150) + +## 汽车票查询 +> python搜索汽车票 + +blog：[python搜索汽车票](http://blog.51cto.com/kaliarch/2071288) + +## 电影爬取 +> 爬取电影并存储到excel + +blog：[爬取搜索出来的电影的下载地址并保存到excel](http://blog.51cto.com/kaliarch/2069544) + +## 爬取推荐博客 +> 爬取推进博客 + +blog：[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto.com/kaliarch/2067103) + From bd7a895f9a0c2c3cb13ae3b6e3085eb3514615c1 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Wed, 7 Mar 2018 16:21:53 +0800 Subject: [PATCH 02/21] change README --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index de6a825..131811d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,25 @@ # my-python-code -## 翻译小工具 +## 翻译小工具(FANYI) > Python实现翻译小工具 blog：[Python实现翻译小工具](http://blog.51cto.com/kaliarch/2072150) -## 汽车票查询 +## 汽车票查询(XAGLKP) > python搜索汽车票 blog：[python搜索汽车票](http://blog.51cto.com/kaliarch/2071288) -## 电影爬取 +## 电影爬取(DYTT8) > 爬取电影并存储到excel blog：[爬取搜索出来的电影的下载地址并保存到excel](http://blog.51cto.com/kaliarch/2069544) -## 爬取推荐博客 +## 爬取推荐博客(51BLOG) > 爬取推进博客 blog：[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto.com/kaliarch/2067103) +## 构建自己的代理库(PROXIES) +> Python构建自己的代理库 + +blog：[Python构建自己的代理库]() From 92219abdd0475cbba4266acee9448dc5e7da91ec Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Thu, 8 Mar 2018 15:51:17 +0800 Subject: [PATCH 03/21] change README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 131811d..bac47fe 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,4 @@ blog：[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto. ## 构建自己的代理库(PROXIES) > Python构建自己的代理库 -blog：[Python构建自己的代理库]() +blog：[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) From f89d213fa660752a0ddffc2e3a2100ed56a02c22 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Sun, 8 Apr 2018 15:33:28 +0800 Subject: [PATCH 04/21] add SANGFOROPER --- README.md | 5 ++ SANGFOROPER/Check_scripts.sh | 8 ++++ SANGFOROPER/Ping_check.sh | 42 +++++++++++++++++ SANGFOROPER/SangFor_oper.py | 90 ++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100644 SANGFOROPER/Check_scripts.sh create mode 100644 SANGFOROPER/Ping_check.sh create mode 100644 SANGFOROPER/SangFor_oper.py diff --git a/README.md b/README.md index bac47fe..df76096 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,9 @@ # my-python-code +## 重启深信服(SANGFOROPER) +> Python深信服重启 + +blog：[Python重启深信服设备](http://blog.51cto.com/kaliarch/2095178) + ## 翻译小工具(FANYI) > Python实现翻译小工具 diff --git a/SANGFOROPER/Check_scripts.sh b/SANGFOROPER/Check_scripts.sh new file mode 100644 index 0000000..a31edf8 --- /dev/null +++ b/SANGFOROPER/Check_scripts.sh @@ -0,0 +1,8 @@ +#!/bin/bash +num=$(ps -ef |grep pdc.sh|wc -l) +cmd="/usr/bin/nohup /bin/bash /sangfor/Shscripts/pdc/Ping_check.sh &" + +if [ ${num} -lt 2 ];then +${cmd} +fi + diff --git a/SANGFOROPER/Ping_check.sh b/SANGFOROPER/Ping_check.sh new file mode 100644 index 0000000..7ce9231 --- /dev/null +++ b/SANGFOROPER/Ping_check.sh @@ -0,0 +1,42 @@ +#!/bin/bash +IP=10.10.10.2 +dir="/sangfor/Shscripts/pdc/" +if [ ! -d ${dir} ];then + mkdir -p ${dir} +fi +echo 1 > ${dir}pdcping.lock +while true +do + Time=`date +%F` + TIME="${Time} 23:59" + if [ "${data}" == "${TIME}" ];then + mkdir ${dir}${Time} && mv ${dir}pdcping.log ${dir}${Time}-pingpdc.log + mv ${dir}${Time}-pingpdc.log ${dir}${Time} + fi + find ${dir} -mtime +7 -type d -exec rm -rf {} \; + + find ${dir} -mtime +7 -name "*-pingpdc.log" -exec rm -rf {} \; + data=`date +%F' '%H:%M` + data1=`date +%F' '%H:%M:%S` + echo "------------${data1}---------------">>${dir}pingpdc.log + ping -c 10 ${IP} >>${dir}pingpdc.log + if [ $? -eq 1 ];then + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 1 ];then + /usr/local/python34/bin/python3 /sangfor/Pysangfor/SangFor_check.py + echo 0 > ${dir}pdcping.lock + else + continue + fi + else + STAT=`cat ${dir}pdcping.lock` + if [ ${STAT} -eq 0 ];then + echo 1 > ${dir}pdcping.lock + else + continue + fi + fi + + +done + diff --git a/SANGFOROPER/SangFor_oper.py b/SANGFOROPER/SangFor_oper.py new file mode 100644 index 0000000..cbfb72a --- /dev/null +++ b/SANGFOROPER/SangFor_oper.py @@ -0,0 +1,90 @@ +#!/bin/env python3 +from pyvirtualdisplay import Display +from selenium import webdriver +import time +import os +import logging + +class Glp_SangFor: + def __init__(self,logger): + self.logger = logger + self.logger.info("--------------start log----------------") + self.display = Display(visible=0, size=(800, 600)) + self.display.start() + self.browser = webdriver.Firefox() + self.logger.info("start browser successfuly") + self.sangfor_url = "深信服url" + self.username = '深信服用户名' + self.password = '深信服密码' + + def login(self): + self.browser.get(self.sangfor_url) + self.browser.implicitly_wait(5) + self.browser.find_element_by_name('user').send_keys(self.username) + self.browser.find_element_by_name('password').send_keys(self.password) + self.browser.find_element_by_class_name('buttons').click() + self.browser.implicitly_wait(5) + self.logger.info("loggin sangfor successfuly") + + def client_reboot(self): + self.browser.find_element_by_id("ext-gen111").click() + print(self.browser.find_element_by_id("ext-gen111").text) + self.browser.implicitly_wait(15) + time.sleep(60) + self.logger.info("switch mainiframe start") + try: + print(self.browser.find_element_by_link_text("重启/重启服务/关机").text) + self.browser.find_element_by_link_text("重启/重启服务/关机").click() + self.browser.implicitly_wait(3) + self.browser.switch_to_frame("mainiframe") + self.browser.implicitly_wait(8) + time.sleep(10) + self.browser.find_element_by_xpath("//button[@id='ext-gen19']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen19']").text) + self.browser.implicitly_wait(10) + #self.browser.find_element_by_xpath("//button[@id='ext-gen42']").click() + print(self.browser.find_element_by_xpath("//button[@id='ext-gen42']").text) + except Exception as e: + self.logger.exception("reboot successful") + return 1 + self.browser.close() + self.logger.info("browser close successful") + self.logger.info("--------------end log----------------") + return 0 + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == '__main__': + os.system("pkill firefox") + os.system("pkill Xvfb") + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + sangfor_oper = Glp_SangFor(logger) + sangfor_oper.login() + sangfor_oper.client_reboot() From 6da4a70a780a743a0eb755923e583bba5b62da87 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Thu, 12 Apr 2018 10:41:01 +0800 Subject: [PATCH 05/21] change SANGFOROPER --- SANGFOROPER/Ecs_apireboot.py | 81 ++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 SANGFOROPER/Ecs_apireboot.py diff --git a/SANGFOROPER/Ecs_apireboot.py b/SANGFOROPER/Ecs_apireboot.py new file mode 100644 index 0000000..a6610c2 --- /dev/null +++ b/SANGFOROPER/Ecs_apireboot.py @@ -0,0 +1,81 @@ +#!/bin/env python3 +# -*- coding:UTF-8 -*- +# _author:kaliarch +from aliyunsdkcore import client +from aliyunsdkecs.request.v20140526 import RebootInstanceRequest,StartInstanceRequest,StopInstanceRequest +import time +import os +import logging +class ecsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('', '', 'cn-hangzhou') + self.logger = logger + self.logger.info("------------------------start reboot vpn ecs of API log-------------") + def reboot_instance(self): + # 设置参数 + request = RebootInstanceRequest.RebootInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn reboot successful!") + self.logger.info(response) + print(response) + + def start_instance(self): + request = StartInstanceRequest.StartInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info("public ecs vpn start successful!") + self.logger.info(response) + print(response) + + def stop_instance(self): + request = StopInstanceRequest.StopInstanceRequest() + request.set_accept_format('json') + request.add_query_param('InstanceId', 'i-bjk23j1rlvfghlq79au') + request.add_query_param('ForceStop', 'false') + # 发起请求 + response = self.clentoper.do_action_with_exception(request) + self.logger.info(response) + print(response) + + def testlog(self): + self.logger.info("public test log") + +class Glp_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'publiclog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + print(LOGFILENAME) + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Glp_Log('public-vpn.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = ecsOper(logger) + app.reboot_instance() + From e5b7c369673a38203141869f32b8af51961bb468 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Mon, 4 Jun 2018 15:48:11 +0800 Subject: [PATCH 06/21] add rdsbak --- RDSBAK/Rds_backup.py | 69 ++++++++++++++++++++ RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log | 3 + README.md | 7 ++ 3 files changed, 79 insertions(+) create mode 100644 RDSBAK/Rds_backup.py create mode 100644 RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log diff --git a/RDSBAK/Rds_backup.py b/RDSBAK/Rds_backup.py new file mode 100644 index 0000000..9055ebd --- /dev/null +++ b/RDSBAK/Rds_backup.py @@ -0,0 +1,69 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from aliyunsdkcore import client +from aliyunsdkrds.request.v20140815 import CreateBackupRequest +import time +import os +import logging + +class rdsOper(): + def __init__(self,logger): + self.clentoper = client.AcsClient('LTAIhfXlcjyln6tW','GwfAMvR4K2ELmt76184oqLTVgRfAso','cn-shanghai') + self.logger = logger + self.logger.info("------------------------start exec rds backup API log-------------") + def backup_instance(self): + # 设置参数 + request = CreateBackupRequest.CreateBackupRequest() + request.set_accept_format('json') + request.add_query_param('DBInstanceId', 'rm-uf6xv35u1x842y61y') + + #如果为单库备份，可以添加DBName + # request.add_query_param('DBName', 'mydb') + + #BackupMethod为备份方式：Logical：逻辑备份，Physical：物理备份 + request.add_query_param('BackupMethod', 'Physical') + #BackupType为备份类型： Auto：自动计算是全量备份还是增量备份；FullBackup：全量备份。默认值为Auto。 + request.add_query_param('BackupType', 'Auto') + + response = self.clentoper.do_action_with_exception(request) + self.logger.info("rdsbackup mission submission successful!") + self.logger.info(response) + print(response) + + +class Rds_Log: + def __init__(self,filename): + self.filename = filename + def createDir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), 'rdsbackuplog') + print(_LOGDIR) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.filename + print(_LOGNAME) + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + print(LOGFILENAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def createlogger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger + +if __name__ == "__main__": + glploger = Rds_Log('rdsbackup.log') + logfilename = glploger.createDir() + logger = glploger.createlogger(logfilename) + + app = rdsOper(logger) + app.backup_instance() + + diff --git a/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log new file mode 100644 index 0000000..02ab8ae --- /dev/null +++ b/RDSBAK/rdsbackuplog/2018-06-04-rdsbackup.log @@ -0,0 +1,3 @@ +2018-06-04 15:40:41,364 - root - INFO - ------------------------start exec rds backup API log------------- +2018-06-04 15:40:41,767 - root - INFO - rdsbackup mission submission successful! +2018-06-04 15:40:41,767 - root - INFO - b'{"RequestId":"2AA53A09-33D3-4A26-94D3-A745301E3A02"}' diff --git a/README.md b/README.md index df76096..7d096f3 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,10 @@ blog：[利用Python搜索51CTO推荐博客并保存至Excel](http://blog.51cto. > Python构建自己的代理库 blog：[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) + + +## Python自定义阿里云RDS备份策略(RDSBAK) +> Python自定义阿里云RDS备份策略 + +blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/blogger/success/2124609) + From 49d769e36c916e4aff19eb4a21813beea12bcc41 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Mon, 4 Jun 2018 15:49:09 +0800 Subject: [PATCH 07/21] add rdsbak --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d096f3..4d76d00 100644 --- a/README.md +++ b/README.md @@ -33,5 +33,5 @@ blog：[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) ## Python自定义阿里云RDS备份策略(RDSBAK) > Python自定义阿里云RDS备份策略 -blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/blogger/success/2124609) +blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/kaliarch/2124609) From a1b7c8cdf4e3941cd1e0531b2889b265b6aed35f Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Mon, 4 Jun 2018 15:57:51 +0800 Subject: [PATCH 08/21] add rdsbak --- RDSBAK/Rds_backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RDSBAK/Rds_backup.py b/RDSBAK/Rds_backup.py index 9055ebd..d01cddc 100644 --- a/RDSBAK/Rds_backup.py +++ b/RDSBAK/Rds_backup.py @@ -10,14 +10,14 @@ class rdsOper(): def __init__(self,logger): - self.clentoper = client.AcsClient('LTAIhfXlcjyln6tW','GwfAMvR4K2ELmt76184oqLTVgRfAso','cn-shanghai') + self.clentoper = client.AcsClient('LTAIhfXlcjyl****','GwfAM344K2ELm345184356TVgRfAso','cn-shanghai') self.logger = logger self.logger.info("------------------------start exec rds backup API log-------------") def backup_instance(self): # 设置参数 request = CreateBackupRequest.CreateBackupRequest() request.set_accept_format('json') - request.add_query_param('DBInstanceId', 'rm-uf6xv35u1x842y61y') + request.add_query_param('DBInstanceId', 'rm-uf6x**5u1x842y61y') #如果为单库备份，可以添加DBName # request.add_query_param('DBName', 'mydb') From 14f4c13913e1d747e0f26178c3849153f0b9f8d6 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Wed, 4 Jul 2018 14:36:09 +0800 Subject: [PATCH 09/21] add rdsbakup --- .../__pycache__/get_rdsinfo.cpython-36.pyc | Bin 0 -> 2247 bytes .../__pycache__/set_log.cpython-36.pyc | Bin 0 -> 1360 bytes get_RdsBakupDownloadUrl/get_rdsinfo.py | 73 ++++++++++++++++++ get_RdsBakupDownloadUrl/info.conf | 30 +++++++ get_RdsBakupDownloadUrl/main.py | 31 ++++++++ .../rdsbacklog/2018-07-04-desrdslog.log | 2 + get_RdsBakupDownloadUrl/set_log.py | 33 ++++++++ 7 files changed, 169 insertions(+) create mode 100644 get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc create mode 100644 get_RdsBakupDownloadUrl/__pycache__/set_log.cpython-36.pyc create mode 100644 get_RdsBakupDownloadUrl/get_rdsinfo.py create mode 100644 get_RdsBakupDownloadUrl/info.conf create mode 100644 get_RdsBakupDownloadUrl/main.py create mode 100644 get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log create mode 100644 get_RdsBakupDownloadUrl/set_log.py diff --git a/get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc b/get_RdsBakupDownloadUrl/__pycache__/get_rdsinfo.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6af67a24562364c517d979a31be81f958117f18 GIT binary patch literal 2247 zcmZuyOOG2x5bmD0XUF?U!e*05V3303LspxY2v!u4>_#YpEURn~Nwd*tGTmP69nYBV zHXB=BF3y1yzkn+TegP5^e`Kz}fgiw~s-Cf9MVV38RQ0pEzOQO-Hk;wS7k9t)RtWiv zEIl^ppFvX(KybooMh28oin^6q1Dn}Kw=;)1&}U~ZbIsRd-sgln+mA#kZL6V+~hU$sE;I=sO>?!TtsK>%lkr7yYqYA9rC)%)N!4zyip>JJc_^y!S8SyQVI+s)u&Q;S=d z;3AjEl*D#zImU8o%Qco;Tb{AJ+VYL%*H&PxKsWl~l=N+A4m5WLTD+p8e$;OoxjG}f zF|&Ah5l>xv$Hq^(o`t*g&05LWmFl5B9%gyo)pKuA}_;U4yWa* zfEi&0-Q$7_wubs%ahhjE!oPm<*v&y`Vi zy+NEYoH2M&x?I#MKDx`31PQ1~bU6b%q%vpNvo>RvaG-*|tkZiL5Kn;nrd0Dem_hkr$xrzxyEe zI!FKf{_Ed={&I9MJ}u-4Y^WhXM~}zxfh_tW)knN$_{)mnN5?|PKur_*;kXRq7}1I2 zCTyp+KoB>eAzfX1H|b?~C+jt%x93YziqqjiYGX6hmY@K&2~e9(X`j~Ara*EFi8!-> z=x7;AF06{2^D&ouixD4IRTk;{{a^~I0!c&qKx`gVX#p@v1S-f**Y`l`rTwD;iAf>f+=h-7|>nE4NLZj`7(97gm=G5>I6XlmFn*)$B7 zRrP44#bB5L<{!3KB!-1;fI~4dd8p%VAqR>fh>4asjgL^n=C`4c5)J|&Z_dfdlx z9nJ{OdQPAmduRTS{xgzW&tNB@h(CGph7cy|3p#Y(9>@VPJsygsfdz%Qig{gR;*rc2 zODFC~cdy-H?jtP*ig{I}BxcE^*_tnb6`g%x#zp8ZU5ueBJuXrxU^2o!8>)m{1x>D@ z!2Fk(m}TSP9Qx7`uX?&27}Vt@7-YfRW9CDx(BhS50Vr|SQ<_Ed!_i;nLxn-5q~dLQ zoUhBn^88O=d$kFIcwAxa%UNSdYjSOgnFUBT5MDa7nEFoRQ?8BumDvuK{&@iL_>=Kpzy0#(?>~>v zE`KQPxvTQ3C|BcWm+H)xb2F*N+PH}=UYHm8!beisIUL!M}wXhrSo!~S1-Msm&MAr z06K8SbpmD;_<30sfSC|hfl}|_#4rM7jlE7RYoDuP1x=R1mm~)t$?xzxp{Yw^Ucl))gCnh z`nnE>FbD?f5U>K)h)`fo5|nLJM{mu>RH<8u*`x-1Koj#8pau>2#Ax$z|8DLkboGI| zb9ASFmtgJHh56nruGvfDTt34p9!v{WGAmXy-=3Lj2pz?mqI!np-U$=Ls+KG*|n91cOt z5ppV9;!K^s&8@e2H`_;pgi4-%sg&<1wJh}oo~80VrM|n!0|=#T4*~6M5+9JD$8k+i y$}J_VjqVbNl&4AB4s*C3W#7I>f{te?y9Dv!wW<(`ow>=E>VI2 literal 0 HcmV?d00001 diff --git a/get_RdsBakupDownloadUrl/get_rdsinfo.py b/get_RdsBakupDownloadUrl/get_rdsinfo.py new file mode 100644 index 0000000..db10d3f --- /dev/null +++ b/get_RdsBakupDownloadUrl/get_rdsinfo.py @@ -0,0 +1,73 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from aliyunsdkcore import client +from aliyunsdkrds.request.v20140815 import DescribeBackupsRequest +import json +import datetime +import configparser + + +class rdsOper(): + def __init__(self,logger): + """ + defined start and end time + init clentoper + """ + configoper = configparser.ConfigParser() + configoper.read('info.conf') + self.accessKeyId = configoper['akconfig']['accessKeyId'] + self.accessSecret = configoper['akconfig']['accessSecret'] + self.region = configoper['akconfig']['region'] + self.instanceId = configoper['akconfig']['instanceId'] + self.BackupMode = configoper['akconfig']['BackupMode'] + self.BackupDownURL = configoper['akconfig']['BackDownloadURL'] + self.clent_oper = client.AcsClient(self.accessKeyId, self.accessSecret, self.region) + self.start_time = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") + 'T00:00Z' + self.end_time = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d") + 'T23:59Z' + self.logger = logger + + def des_instance(self): + """ + set param + :return: response + """ + request = DescribeBackupsRequest.DescribeBackupsRequest() + request.set_accept_format('json') + # DBInstanceId: + request.add_query_param('DBInstanceId', self.instanceId) + # StartTime: + request.add_query_param('StartTime', self.start_time) + # EndTime: + request.add_query_param('EndTime', self.end_time) + # BackupId: + # request.add_query_param('BackupId', '备份集ID') + # BackupStatus: + request.add_query_param('BackupStatus', 'Success') + # 备份类型，取值范围:Automated：常规任务 + request.add_query_param('BackupMode', self.BackupMode) + response = self.clent_oper.do_action_with_exception(request) + self.logger.info("init request complate!") + return response + + # 接受响应，定义外网或内网url,公网：BackupDownloadURL，内网：BackupIntranetDownloadURL + def get_rdsdownload_url(self, response, net_type='BackupDownloadURL'): + """ + :param response: + :param net_type: + :return: rds_download_url + """ + context = json.loads(response.decode('utf-8')) + con_list = context['Items']['Backup'] + net_type = self.BackupDownURL + try: + download_url = con_list[0][net_type] + + log_url = str(datetime.datetime.now()) +" downloadurl:"+ download_url + self.logger.info(log_url) + except Exception as e: + self.logger.info(e) + exit() + return download_url + diff --git a/get_RdsBakupDownloadUrl/info.conf b/get_RdsBakupDownloadUrl/info.conf new file mode 100644 index 0000000..836a261 --- /dev/null +++ b/get_RdsBakupDownloadUrl/info.conf @@ -0,0 +1,30 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch +#Get the download address of the RDS backupdownload url, if you want to get other information, you can modify the code yourself. + +[akconfig] +#Configure Ali cloud accesskeyid +accessKeyId = LTAIhfXlcjyln6tW + +#Configure Ali cloud accessscret +accessSecret = GwfAMvR4K2ELmt76184oqLTVgRfAso + +#Configuring the area where the RDS instance is located +region = cn-beijing + +#Instance ID for configuring RDS +instanceId = rm-2zer20742k7bl5b56 + +#Configure RDS backup type type:"Automated" system automatically backup, "Manual" manually perform backup +BackupMode = Manual + +Download URL type configured: type:"BackupIntranetDownloadURL" intranet download address, "BackupDownloadURL" public network download address +BackDownloadURL = BackupDownloadURL + +# loginfo configuration +[logconfig] +#Log file directory name +logdir_name = rdsbacklog +#Log file name +logfile_name = desrdslog.log diff --git a/get_RdsBakupDownloadUrl/main.py b/get_RdsBakupDownloadUrl/main.py new file mode 100644 index 0000000..0ac2169 --- /dev/null +++ b/get_RdsBakupDownloadUrl/main.py @@ -0,0 +1,31 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch + +from get_rdsinfo import rdsOper +from set_log import rdsLog + + +def init_log(): + """ + init logger + :return: logger + """ + logoper = rdsLog() + filename = logoper.create_dir() + rdslog = logoper.create_logger(filename) + return rdslog + +def app_rdsbak(): + """ + + :return: + """ + log = init_log() + rdsapp = rdsOper(log) + result = rdsapp.des_instance() +# print(rdsapp.get_rdsdownload_url(result, 'BackupIntranetDownloadURL')) + print(rdsapp.get_rdsdownload_url(result, 'BackupDownloadURL')) + +if __name__ == "__main__": + app_rdsbak() diff --git a/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log b/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log new file mode 100644 index 0000000..0a401a6 --- /dev/null +++ b/get_RdsBakupDownloadUrl/rdsbacklog/2018-07-04-desrdslog.log @@ -0,0 +1,2 @@ +2018-07-04 13:52:41,389 - root - INFO - init request complate! +2018-07-04 13:52:41,389 - root - INFO - 2018-07-04 13:52:41.389141 downloadurl:http://rdsbak-bjc-v2.oss-cn-beijing-c.aliyuncs.com/custins7231221/hins4953173_data_20180704113854.tar.gz?OSSAccessKeyId=LTAITfQ7krsrEwRn&Expires=1530769960&Signature=1XEZhUjQFCNir6rpd4omo71NyGs%3D diff --git a/get_RdsBakupDownloadUrl/set_log.py b/get_RdsBakupDownloadUrl/set_log.py new file mode 100644 index 0000000..f472afe --- /dev/null +++ b/get_RdsBakupDownloadUrl/set_log.py @@ -0,0 +1,33 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _author:kaliarch +import os +import time +import logging +import configparser + +class rdsLog: + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('info.conf') + self.logdir_name = configoper['logconfig']['logdir_name'] + self.logfile_name = configoper['logconfig']['logfile_name'] + + def create_dir(self): + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self,logfilename): + logger= logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger From 514ffbbfa8fc2dab29e6b2e1cc2690408925e2e7 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Thu, 5 Jul 2018 14:40:06 +0800 Subject: [PATCH 10/21] add requirements.txt --- README.md | 4 ++++ get_RdsBakupDownloadUrl/requirements.txt | 15 +++++++++++++++ requirements.txt | 15 +++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 get_RdsBakupDownloadUrl/requirements.txt create mode 100644 requirements.txt diff --git a/README.md b/README.md index 4d76d00..81f3571 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,7 @@ blog：[Python构建自己的代理库](http://blog.51cto.com/kaliarch/2083997) blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/kaliarch/2124609) +## RDS物理备份恢复至线下IDC服务器实践(RDSBAK) +> RDS物理备份恢复至线下IDC服务器实践 + +blog：[RDS物理备份自动恢复至IDC线下服务器实践](#) diff --git a/get_RdsBakupDownloadUrl/requirements.txt b/get_RdsBakupDownloadUrl/requirements.txt new file mode 100644 index 0000000..13c9dba --- /dev/null +++ b/get_RdsBakupDownloadUrl/requirements.txt @@ -0,0 +1,15 @@ +aliyun-python-sdk-core==2.3.5 +aliyun-python-sdk-core-v3==2.8.6 +aliyun-python-sdk-ecs==4.7.1 +aliyun-python-sdk-rds==2.1.3 +certifi==2018.1.18 +chardet==3.0.4 +EasyProcess==0.2.3 +idna==2.6 +ipython==5.3.0 +pycryptodome==3.6.1 +PyVirtualDisplay==0.2.1 +requests==2.18.4 +selenium==3.4.3 +urllib3==1.22 +XlsxWriter==0.9.6 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..13c9dba --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +aliyun-python-sdk-core==2.3.5 +aliyun-python-sdk-core-v3==2.8.6 +aliyun-python-sdk-ecs==4.7.1 +aliyun-python-sdk-rds==2.1.3 +certifi==2018.1.18 +chardet==3.0.4 +EasyProcess==0.2.3 +idna==2.6 +ipython==5.3.0 +pycryptodome==3.6.1 +PyVirtualDisplay==0.2.1 +requests==2.18.4 +selenium==3.4.3 +urllib3==1.22 +XlsxWriter==0.9.6 From b5868f9610ad8e8f3ada4c738eda76fe3ba72330 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Mon, 23 Jul 2018 23:22:39 +0800 Subject: [PATCH 11/21] add tcp_port_check --- README.md | 5 + tcp_port_check/__init__.py | 5 + .../__pycache__/logger.cpython-36.pyc | Bin 0 -> 1616 bytes tcp_port_check/info.cfg | 19 ++++ .../logdir/2018-07-23-check_port.log | 30 ++++++ tcp_port_check/logger.py | 47 +++++++++ tcp_port_check/tcp_port_check.py | 95 ++++++++++++++++++ 7 files changed, 201 insertions(+) create mode 100644 tcp_port_check/__init__.py create mode 100644 tcp_port_check/__pycache__/logger.cpython-36.pyc create mode 100644 tcp_port_check/info.cfg create mode 100644 tcp_port_check/logdir/2018-07-23-check_port.log create mode 100644 tcp_port_check/logger.py create mode 100644 tcp_port_check/tcp_port_check.py diff --git a/README.md b/README.md index 81f3571..40d9fbf 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,8 @@ blog：[Python自定义阿里云RDS备份策略](http://blog.51cto.com/kaliarch/ > RDS物理备份恢复至线下IDC服务器实践 blog：[RDS物理备份自动恢复至IDC线下服务器实践](#) + +## Python实现端口检测(tcp_port_check) +> Python实现端口检测 + +blog：[Python实现端口检测](http://blog.51cto.com/kaliarch/2149228) diff --git a/tcp_port_check/__init__.py b/tcp_port_check/__init__.py new file mode 100644 index 0000000..a4af0aa --- /dev/null +++ b/tcp_port_check/__init__.py @@ -0,0 +1,5 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + diff --git a/tcp_port_check/__pycache__/logger.cpython-36.pyc b/tcp_port_check/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1ddc3ee7f19c3e126149d5ef2e30fcfda973503 GIT binary patch literal 1616 zcmZux&yU+g6rS<#IN26KQM?&QaFHws$M8uK(O>5nJYCdCr(^A^}X?CcMFW=&+pC5dwKc3H}mq!iu>HR z?_IfT67oAaGIX%dz?;1UzzL@#;#FVjQRQo%roOtTUm#rL`ZdD!h2|NYzC+r^pAbj- z0pfCKV4Q(BTLTCQ=7Aj4)2nERG`M++;U$<&E7@8bzV_>$e+^?(LPgb|2pUzP$fsdF!)3c0YZv_fz@B&4+h>P!R!zSR7b} z0qL`SW)V?tis)`Rd21*vWD~kJDr; z4<>Aqikt=8VX(7RF}6OrDjmk+B+eN-4$+wdK&V5H!83chx8!_%x#@sFtOff+5%>WZ zNZ?RYsOQFl7G!8Hv^mYKq0JGU1zBiZJ4xnRp}}emEWL^^fQ? zf&oP9)6?)HO2kU~+vVP`mCy&1T@mI}k#w8?(H%o(PV4~sqQs*IqM9+es9RncdDSCpsphg35 zcH&Tx!_5IUGY-@QZKX*M4CE7<8u(Vq=*T9_8OZBgFZ3s7O^&k$%vuHL2fS-T8+1_? zD4!?80N9Da=9{u>u!BQ6**&jDy4S7UPyN7E_js7=s_DXBb&jRGPoi#hn z)*43do1^ffaCF2Uhgs%Fu-bMco-;UTk*q~w-c#2BNCqaxNMp0VaY@!Upqh(*!bfn( zY8$B-`+1J%4fY*+uFv`Yrlp*Cc9;J|T2+IV8D?iDyQmJ=X6!h8@R1X2loa% F+GENewH5#X literal 0 HcmV?d00001 diff --git a/tcp_port_check/info.cfg b/tcp_port_check/info.cfg new file mode 100644 index 0000000..631d5ad --- /dev/null +++ b/tcp_port_check/info.cfg @@ -0,0 +1,19 @@ +#公网端口检测配置 +[port_check_info] + +#检测ip地址或域名 +#address = baidu.com +#address = 8.8.8.8 +address = www.51cto.com,www.anchnet.com + +#检查的端口，如多个端口使用,隔开,端口范围使用'-' +#ports = 80,8080.... +ports = 20-25,80,443,1433,1521,3306,3389,6379,8080,27017 + +#日志配置 +[loginfo] +#日志目录 +logdir_name = logdir + +#日志文件名称 +logfile_name = check_port.log \ No newline at end of file diff --git a/tcp_port_check/logdir/2018-07-23-check_port.log b/tcp_port_check/logdir/2018-07-23-check_port.log new file mode 100644 index 0000000..23303c1 --- /dev/null +++ b/tcp_port_check/logdir/2018-07-23-check_port.log @@ -0,0 +1,30 @@ +2018-07-23 23:07:55,506 - root - INFO - www.51cto.com:20,port status is:['ر'] +2018-07-23 23:07:56,985 - root - INFO - www.51cto.com:21,port status is:['ر'] +2018-07-23 23:07:58,705 - root - INFO - www.51cto.com:22,port status is:['ر'] +2018-07-23 23:08:00,786 - root - INFO - www.51cto.com:23,port status is:['ر'] +2018-07-23 23:08:02,866 - root - INFO - www.51cto.com:24,port status is:['ر'] +2018-07-23 23:08:04,383 - root - INFO - www.51cto.com:25,port status is:['ر'] +2018-07-23 23:08:09,746 - root - INFO - www.anchnet.com:20,port status is:['ر'] +2018-07-23 23:08:15,187 - root - INFO - www.anchnet.com:21,port status is:['ر'] +2018-07-23 23:08:20,626 - root - INFO - www.anchnet.com:22,port status is:['ر'] +2018-07-23 23:08:25,917 - root - INFO - www.anchnet.com:23,port status is:['ر'] +2018-07-23 23:08:31,506 - root - INFO - www.anchnet.com:24,port status is:['ر'] +2018-07-23 23:08:37,585 - root - INFO - www.anchnet.com:25,port status is:['ر'] +2018-07-23 23:08:38,385 - root - INFO - www.51cto.com:80,port status is:[''] +2018-07-23 23:08:38,903 - root - INFO - www.51cto.com:443,port status is:[''] +2018-07-23 23:08:40,946 - root - INFO - www.51cto.com:1433,port status is:['ر'] +2018-07-23 23:08:42,705 - root - INFO - www.51cto.com:1521,port status is:['ر'] +2018-07-23 23:08:45,002 - root - INFO - www.51cto.com:3306,port status is:['ر'] +2018-07-23 23:08:47,026 - root - INFO - www.51cto.com:3389,port status is:['ر'] +2018-07-23 23:08:48,627 - root - INFO - www.51cto.com:6379,port status is:['ر'] +2018-07-23 23:08:49,291 - root - INFO - www.51cto.com:8080,port status is:[''] +2018-07-23 23:08:50,788 - root - INFO - www.51cto.com:27017,port status is:['ر'] +2018-07-23 23:08:51,668 - root - INFO - www.anchnet.com:80,port status is:[''] +2018-07-23 23:08:52,751 - root - INFO - www.anchnet.com:443,port status is:[''] +2018-07-23 23:08:58,148 - root - INFO - www.anchnet.com:1433,port status is:['ر'] +2018-07-23 23:09:05,105 - root - INFO - www.anchnet.com:1521,port status is:['ر'] +2018-07-23 23:09:10,555 - root - INFO - www.anchnet.com:3306,port status is:['ر'] +2018-07-23 23:09:16,032 - root - INFO - www.anchnet.com:3389,port status is:['ر'] +2018-07-23 23:09:22,466 - root - INFO - www.anchnet.com:6379,port status is:['ر'] +2018-07-23 23:09:28,314 - root - INFO - www.anchnet.com:8080,port status is:['ر'] +2018-07-23 23:09:34,545 - root - INFO - www.anchnet.com:27017,port status is:['ر'] diff --git a/tcp_port_check/logger.py b/tcp_port_check/logger.py new file mode 100644 index 0000000..c2e4095 --- /dev/null +++ b/tcp_port_check/logger.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + +import os +import time +import logging +import configparser + +class LogHelper(): + """ + 初始化logger，读取目录及文件名称 + """ + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('info.cfg',encoding='utf-8') + self.logdir_name = configoper['loginfo']['logdir_name'] + self.logfile_name = configoper['loginfo']['logfile_name'] + + def create_dir(self): + """ + 创建目录 + :return: 文件名称 + """ + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self, logfilename): + """ + 创建logger对象 + :param logfilename: + :return: logger对象 + """ + logger = logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger diff --git a/tcp_port_check/tcp_port_check.py b/tcp_port_check/tcp_port_check.py new file mode 100644 index 0000000..1ee732c --- /dev/null +++ b/tcp_port_check/tcp_port_check.py @@ -0,0 +1,95 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +from configparser import ConfigParser +import re +import logger + +class check_ports(): + def __init__(self,logger): + """ + 初始化，获取配置文件信息 + """ + self.url = 'http://tool.chinaz.com/iframe.ashx?t=port' + self.headers = { + 'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.8', + 'Connection': 'keep-alive', + 'Content-Length': '62', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Host': 'tool.chinaz.com', + 'Origin': 'http://tool.chinaz.com', + 'Referer': 'http://tool.chinaz.com/port/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36', + 'X-Requested-With': 'XMLHttpRequest' + } + config = ConfigParser() + config.read('info.cfg',encoding='utf-8') + self.address_list = config['port_check_info']['address'] + self.port_list = config['port_check_info']['ports'] + #初始化logger + logger = logger.LogHelper() + logname = logger.create_dir() + self.logoper = logger.create_logger(logname) + + + def _get_body(self): + """ + 获取address和port + :return: list + """ + address_list = self.address_list.split(',') + port_list = self.port_list.split(',') + + # 处理端口范围，返回range + range_flag = False + port_range = None + content_List_range = [] + for port in port_list: + if '-' in port: + range_flag = True + port_range = range(int(port.split('-')[0]),int(port.split('-')[1])+1) + port_list.remove(port) + + # 处理总体list + for add in address_list: + if range_flag: + for port in port_range: + content_List_range.append(add + ':' + str(port)) + + # 合并range和普通list + content_List = [ add+':'+port for add in address_list for port in port_list ] + content_List_range.extend(content_List) + return content_List_range + + def run(self): + """ + 进行端口检测 + :return: + """ + for content in self._get_body(): + content_list = content.split(':') + body = { + 'host': content_list[0], + 'port': content_list[1], + 'encode': 'tlCHS1u3IgF4sC57m6KOP3Oaj1Y1kfLq' + } + try: + response = requests.post(url=self.url,data=body,headers=self.headers) + port_status = re.findall("msg:'(.*?)'", response.text) + if len(port_status) > 0: + print('%s,port status is:%s' % (content, port_status)) + self.logoper.info('%s,port status is:%s' % (content, port_status)) + else: + self.logoper.info('%s,port status is:%s' % (content, port_status)) + print('Occer error！请输入正确的地址和端口') + except Exception as e: + self.logoper.info(e) + + +if __name__ == '__main__': + check_app = check_ports(logger) + check_app.run() \ No newline at end of file From 66558c23bd2841fe6ccd1d884176a337bf60fede Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Wed, 25 Jul 2018 16:42:49 +0800 Subject: [PATCH 12/21] add imageoper --- README.md | 5 + imageoper/__pycache__/logger.cpython-36.pyc | Bin 0 -> 1622 bytes imageoper/config.cfg | 38 +++++++ imageoper/create_cop_image.py | 120 ++++++++++++++++++++ imageoper/logdir/2018-07-25-ecsoperlog.log | 3 + imageoper/logger.py | 47 ++++++++ 6 files changed, 213 insertions(+) create mode 100644 imageoper/__pycache__/logger.cpython-36.pyc create mode 100644 imageoper/config.cfg create mode 100644 imageoper/create_cop_image.py create mode 100644 imageoper/logdir/2018-07-25-ecsoperlog.log create mode 100644 imageoper/logger.py diff --git a/README.md b/README.md index 40d9fbf..999801d 100644 --- a/README.md +++ b/README.md @@ -44,3 +44,8 @@ blog：[RDS物理备份自动恢复至IDC线下服务器实践](#) > Python实现端口检测 blog：[Python实现端口检测](http://blog.51cto.com/kaliarch/2149228) + +## Python实现ECS自动镜像创建&镜像复制至其他地域(imageoper) +> Python实现ECS自动镜像创建&镜像复制至其他地域 + +blog：[Python实现ECS自动镜像创建&镜像复制至其他地域](http://blog.51cto.com/kaliarch/2150076) diff --git a/imageoper/__pycache__/logger.cpython-36.pyc b/imageoper/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db1d2b9f4554b3278a3dd4adc2f96681e40dac5c GIT binary patch literal 1622 zcmZux&yU+g6rS<#{3;b%RF+eOh_pFm4}hSGD%4fmZ6k$r1suRc9a$d7>om55?Eo7& zr&OYXR^`H8Dh`~gUMO6UVBsH`E0~rO_nvrfoa}B}Z26n_X8fL?zwgbw)o4@~XFvY* zp-#x3 zp#k0M>;im5N?A6*{o?4iQtso)ZsKLLByLP z(8h5P2@^Du5&zGs!*kEvi|j} zmwO-BI{T>i{uSFof3O2q4UFAh{|Y)%xf1FSC8{pyop}rry#juE27u_V(hXV#HR^!# zjT1#qUUspi@j^||mYVdzKt7?hfp4jdifqE1hP*D+TzzI%=QyjutdWC$z`HUwK^JL( z0?G_!J|?@kjS7@~R6P3i$LS1GwApF4 zwlI2SPQ1^($tged($w)`wQUI#)8VG8_+HkRhXP0jCdLS@H`u){YP(R)H7DW|*lCsB zI0>CB!%YY8op@D;^VLghIdKE8|3nyN;lkW6GuhTiX7ZBf^=zypDg#~z^3t;a#IU58 zC@9|Zb5L)2z->PhF=(5(JMoVh6II5-n9tzkV4}|0XER4~0iMYik6p%+1|&~55#B(+ zgW(ilk0sb1I4?K4rE1hNtcIM0pT!d(*+6&=!9>7-Q%Ph_*{#-vvYhwuXe!gexdFHG EA2uhq3jhEB literal 0 HcmV?d00001 diff --git a/imageoper/config.cfg b/imageoper/config.cfg new file mode 100644 index 0000000..60a14a9 --- /dev/null +++ b/imageoper/config.cfg @@ -0,0 +1,38 @@ +# 阿里云ak配置，建议采用子账户只授权ecs镜像操作 +[common] +# 阿里云acccesskeyid +accessKeyId = LTAIhfXlcxxxxxxxxx +# 阿里云accesssecret +accessSecret = GwfAMvR4K2ELmtxxxxxxxxxxxxxx +# log目录名称 +logdir_name = logdir +# log文件名称 +logfile_name = ecsoperlog.log + +# ecs源地域配置信息段 +#支持在华北 1、华北 2、华北 3、华北 5、华东 1、华东 2 和华南 1 地域之间复制镜像。涉及其他国家和地区地域时，可以 提交工单 申请 +[source] +# 源地域实例regionid，可以参考：https://help.aliyun.com/document_detail/40654.html?spm=a2c1g.8271268.10000.5.5f98df25B98bhJ +s_RegionId = cn-shanghai + +# 源实例id +s_InstanceId = i-uf6er9hxb1vt0bxgjctw + +# 源端制作镜像name +s_ImageName = api-source-image + +# 源镜像描述信息 +s_Description = api-source-image源镜像描述信息 + +# 镜像复制目的地域配置信息段 +[destination] +# 目的地域实例regionid， +d_DestinationRegionId = cn-qingdao + +# 复制过来的镜像名称 +d_DestinationImageName = api-destination-image + +# 复制过来的镜像描述信息 +d_DestinationDescription = api-destination-image目的镜像描述信息 + + diff --git a/imageoper/create_cop_image.py b/imageoper/create_cop_image.py new file mode 100644 index 0000000..93fa562 --- /dev/null +++ b/imageoper/create_cop_image.py @@ -0,0 +1,120 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import configparser +import time +import json +from aliyunsdkcore import client +from aliyunsdkecs.request.v20140526 import CreateImageRequest +from aliyunsdkecs.request.v20140526 import CopyImageRequest +from aliyunsdkecs.request.v20140526 import DescribeImagesRequest +import logger + + +class EcsOper(): + def __init__(self,logger): + """ + 初始化获取系config信息，配置日志 + :param logger: + """ + config = configparser.ConfigParser() + config.read('config.cfg',encoding='utf-8') + accessKeyId = config['common']['accessKeyId'] + accessSecret = config['common']['accessSecret'] + self.s_RegionId = config['source']['s_RegionId'] + self.s_InstanceId = config['source']['s_InstanceId'] + + self.s_ImageName = config['source']['s_ImageName'] + self.s_Description = config['source']['s_Description'] + + self.d_DestinationRegionId = config['destination']['d_DestinationRegionId'] + self.d_DestinationImageName = config['destination']['d_DestinationImageName'] + self.d_DestinationDescription = config['destination']['d_DestinationDescription'] + self.ecshelper = client.AcsClient(accessKeyId,accessSecret,self.s_RegionId) + + logger = logger.LogHelper() + logname = logger.create_dir() + self.logoper = logger.create_logger(logname) + + # 镜像制作 + def _create_image(self): + """ + 创建镜像 + :return:返回镜像id + """ + s_timer = time.strftime("%Y-%m-%d-%H:%M", time.localtime(time.time())) + request = CreateImageRequest.CreateImageRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('InstanceId', self.s_InstanceId) + request.add_query_param('ImageName', self.s_ImageName + s_timer) + request.add_query_param('Description', self.s_Description + s_timer) + response = self.ecshelper.do_action_with_exception(request) + self.logoper.info('创建镜像任务已提交,镜像id:%s' % json.loads(response)["ImageId"]) + print('创建镜像任务已提交,镜像id:%s' % json.loads(response)["ImageId"]) + return json.loads(response)["ImageId"] + + # 查询镜像状态 + def _describe_image(self,imageid): + """ + 查询image状态 + :param imageid: + :return: + """ + request = DescribeImagesRequest.DescribeImagesRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('ImageId', imageid) + response = self.ecshelper.do_action_with_exception(request) + # 进度 json.loads(response)['Images']['Image'][0]['Progress'] + self.logoper.info('镜像创建进度:%s' %json.loads(response)['Images']['Image'][0]['Progress']) + # 镜像状态 + return json.loads(response)['Images']['Image'][0]['Status'] + + + #镜像复制 + def _copy_image(self,imageid): + """ + 镜像复制 + :param imageid:源镜像id + :return: 复制成功后的镜像id + """ + flag = True + while flag: + try: + if self._describe_image(imageid) == 'Available': + flag = False + else: + time.sleep(300) + except Exception as e: + pass + print('镜像已经创建完成') + d_timer = time.strftime("%Y-%m-%d-%H:%M", time.localtime(time.time())) + request = CopyImageRequest.CopyImageRequest() + request.set_accept_format('json') + request.add_query_param('RegionId', self.s_RegionId) + request.add_query_param('DestinationRegionId', self.d_DestinationRegionId) + request.add_query_param('DestinationImageName', self.d_DestinationImageName + d_timer) + request.add_query_param('DestinationDescription', self.d_DestinationDescription + d_timer) + request.add_query_param('ImageId', imageid) + response = self.ecshelper.do_action_with_exception(request) + self.logoper.info('复制镜像任务已提交,镜像id:%s' % json.loads(response)['ImageId']) + print('复制镜像任务已提交,镜像id:%s' % json.loads(response)['ImageId']) + return json.loads(response)['ImageId'] + + + def run(self): + s_imageid = self._create_image() + self._copy_image(s_imageid) + + +if __name__ == '__main__': + ecsoper = EcsOper(logger) + ecsoper.run() + + + + + + diff --git a/imageoper/logdir/2018-07-25-ecsoperlog.log b/imageoper/logdir/2018-07-25-ecsoperlog.log new file mode 100644 index 0000000..f2c3e93 --- /dev/null +++ b/imageoper/logdir/2018-07-25-ecsoperlog.log @@ -0,0 +1,3 @@ +2018-07-25 16:13:07,098 - root - INFO - ύ,id:m-uf6eby4dmp9llm9dgz50 +2018-07-25 16:18:09,721 - root - INFO - 񴴽:100% +2018-07-25 16:18:10,067 - root - INFO - ƾύ,id:m-m5eb53zrph7pic6zuejq diff --git a/imageoper/logger.py b/imageoper/logger.py new file mode 100644 index 0000000..338d1dd --- /dev/null +++ b/imageoper/logger.py @@ -0,0 +1,47 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + + +import os +import time +import logging +import configparser + +class LogHelper(): + """ + 初始化logger，读取目录及文件名称 + """ + def __init__(self): + configoper = configparser.ConfigParser() + configoper.read('config.cfg',encoding='utf-8') + self.logdir_name = configoper['common']['logdir_name'] + self.logfile_name = configoper['common']['logfile_name'] + + def create_dir(self): + """ + 创建目录 + :return: 文件名称 + """ + _LOGDIR = os.path.join(os.path.dirname(__file__), self.logdir_name) + _TIME = time.strftime('%Y-%m-%d', time.gmtime()) + '-' + _LOGNAME = _TIME + self.logfile_name + LOGFILENAME = os.path.join(_LOGDIR, _LOGNAME) + if not os.path.exists(_LOGDIR): + os.mkdir(_LOGDIR) + return LOGFILENAME + + def create_logger(self, logfilename): + """ + 创建logger对象 + :param logfilename: + :return: logger对象 + """ + logger = logging.getLogger() + logger.setLevel(logging.INFO) + handler = logging.FileHandler(logfilename) + handler.setLevel(logging.INFO) + formater = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formater) + logger.addHandler(handler) + return logger From ca2c08c4ca53b3e787800055ab780fa98496160f Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Thu, 26 Jul 2018 10:34:20 +0800 Subject: [PATCH 13/21] change imageoper --- imageoper/config.cfg | 8 ++++---- imageoper/create_cop_image.py | 22 ++++++++++++---------- imageoper/logdir/2018-07-26-ecsoperlog.log | 6 ++++++ 3 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 imageoper/logdir/2018-07-26-ecsoperlog.log diff --git a/imageoper/config.cfg b/imageoper/config.cfg index 60a14a9..3270493 100644 --- a/imageoper/config.cfg +++ b/imageoper/config.cfg @@ -1,9 +1,9 @@ # 阿里云ak配置，建议采用子账户只授权ecs镜像操作 [common] # 阿里云acccesskeyid -accessKeyId = LTAIhfXlcxxxxxxxxx +accessKeyId = LTAIhfXlcjyln6tW # 阿里云accesssecret -accessSecret = GwfAMvR4K2ELmtxxxxxxxxxxxxxx +accessSecret = GwfAMvR4K2ELmt76184oqLTVgRfAso # log目录名称 logdir_name = logdir # log文件名称 @@ -15,8 +15,8 @@ logfile_name = ecsoperlog.log # 源地域实例regionid，可以参考：https://help.aliyun.com/document_detail/40654.html?spm=a2c1g.8271268.10000.5.5f98df25B98bhJ s_RegionId = cn-shanghai -# 源实例id -s_InstanceId = i-uf6er9hxb1vt0bxgjctw +# 源实例id,可指定多个用，进行分隔 +s_InstanceId = i-uf661wb708uvqc9jyhem,i-uf661wb708uvqc9jyhel # 源端制作镜像name s_ImageName = api-source-image diff --git a/imageoper/create_cop_image.py b/imageoper/create_cop_image.py index 93fa562..9a948c3 100644 --- a/imageoper/create_cop_image.py +++ b/imageoper/create_cop_image.py @@ -23,8 +23,7 @@ def __init__(self,logger): accessKeyId = config['common']['accessKeyId'] accessSecret = config['common']['accessSecret'] self.s_RegionId = config['source']['s_RegionId'] - self.s_InstanceId = config['source']['s_InstanceId'] - + self.s_InstanceId_list = config['source']['s_InstanceId'] self.s_ImageName = config['source']['s_ImageName'] self.s_Description = config['source']['s_Description'] @@ -37,8 +36,13 @@ def __init__(self,logger): logname = logger.create_dir() self.logoper = logger.create_logger(logname) + # 创建实例生成器 + def _get_Instance(self): + for Instance in self.s_InstanceId_list.split(','): + yield Instance + # 镜像制作 - def _create_image(self): + def _create_image(self,s_InstanceId): """ 创建镜像 :return:返回镜像id @@ -47,12 +51,12 @@ def _create_image(self): request = CreateImageRequest.CreateImageRequest() request.set_accept_format('json') request.add_query_param('RegionId', self.s_RegionId) - request.add_query_param('InstanceId', self.s_InstanceId) + request.add_query_param('InstanceId', s_InstanceId) request.add_query_param('ImageName', self.s_ImageName + s_timer) request.add_query_param('Description', self.s_Description + s_timer) response = self.ecshelper.do_action_with_exception(request) self.logoper.info('创建镜像任务已提交,镜像id:%s' % json.loads(response)["ImageId"]) - print('创建镜像任务已提交,镜像id:%s' % json.loads(response)["ImageId"]) + print('实例%s,创建镜像任务已提交,镜像id:%s' % (s_InstanceId,json.loads(response)["ImageId"])) return json.loads(response)["ImageId"] # 查询镜像状态 @@ -72,7 +76,6 @@ def _describe_image(self,imageid): # 镜像状态 return json.loads(response)['Images']['Image'][0]['Status'] - #镜像复制 def _copy_image(self,imageid): """ @@ -103,11 +106,10 @@ def _copy_image(self,imageid): print('复制镜像任务已提交,镜像id:%s' % json.loads(response)['ImageId']) return json.loads(response)['ImageId'] - def run(self): - s_imageid = self._create_image() - self._copy_image(s_imageid) - + for instance_id in self._get_Instance(): + s_imageid = self._create_image(instance_id) + self._copy_image(s_imageid) if __name__ == '__main__': ecsoper = EcsOper(logger) diff --git a/imageoper/logdir/2018-07-26-ecsoperlog.log b/imageoper/logdir/2018-07-26-ecsoperlog.log new file mode 100644 index 0000000..9fe16ff --- /dev/null +++ b/imageoper/logdir/2018-07-26-ecsoperlog.log @@ -0,0 +1,6 @@ +2018-07-26 10:16:59,150 - root - INFO - ύ,id:m-uf6cp52t42s72i4jgxli +2018-07-26 10:22:00,824 - root - INFO - 񴴽:100% +2018-07-26 10:22:01,314 - root - INFO - ƾύ,id:m-m5e8b39oaqnhbfp9x35i +2018-07-26 10:22:03,383 - root - INFO - ύ,id:m-uf60h59flke4cqhtg6nn +2018-07-26 10:27:04,725 - root - INFO - 񴴽:100% +2018-07-26 10:27:05,060 - root - INFO - ƾύ,id:m-m5ednmzrres9mkjjsq0l From 42c383b3e97e0a1fd7b5bde3a54d20401f7eaa05 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Tue, 21 Aug 2018 11:57:05 +0800 Subject: [PATCH 14/21] add doutulai --- .../multithreading_spider/dutulai_spider.py | 117 ++++++++++++++++++ doutulai/scrapy_doutulai/scrapy.cfg | 11 ++ .../scrapy_doutulai/__init__.py | 0 .../scrapy_doutulai/scrapy_doutulai/items.py | 17 +++ .../scrapy_doutulai/scrapy_doutulai/main.py | 8 ++ .../scrapy_doutulai/middlewares.py | 103 +++++++++++++++ .../scrapy_doutulai/pipelines.py | 31 +++++ .../scrapy_doutulai/settings.py | 93 ++++++++++++++ .../scrapy_doutulai/spiders/__init__.py | 4 + .../scrapy_doutulai/spiders/__init__.pyc | Bin 0 -> 180 bytes .../spiders/doutulai_spider.py | 28 +++++ .../spiders/doutulai_spider.pyc | Bin 0 -> 1517 bytes 12 files changed, 412 insertions(+) create mode 100644 doutulai/multithreading_spider/dutulai_spider.py create mode 100644 doutulai/scrapy_doutulai/scrapy.cfg create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/__init__.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/items.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/main.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/settings.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.py create mode 100644 doutulai/scrapy_doutulai/scrapy_doutulai/spiders/doutulai_spider.pyc diff --git a/doutulai/multithreading_spider/dutulai_spider.py b/doutulai/multithreading_spider/dutulai_spider.py new file mode 100644 index 0000000..d990548 --- /dev/null +++ b/doutulai/multithreading_spider/dutulai_spider.py @@ -0,0 +1,117 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +from bs4 import BeautifulSoup +import urllib +import os +import threading + +class DutuSpider(): + # 定义全局页面url列表 + page_url_list = [] + # 定义具体各表情图片url列表 + img_url_list = [] + # 定义rlock进程锁 + rlock = threading.RLock() + + def __init__(self,page_number=10,img_dir='imgdir',thread_number=5): + """ + :param page_number: 抓去多少个页面，默认10 + :param img_dir: 定义图片目录 + :param thread_number:默认5个线程 + """ + self.spider_url = 'https://www.doutula.com/photo/list/?page=' + self.page_number = int(page_number) + self.img_dir = img_dir + self.thread_num = thread_number + + + def get_url(self): + """ + 创建image目录和生产pageurl列表 + :return: + """ + if not os.path.exists(self.img_dir): + os.makedirs(self.img_dir) + for page in range(1,self.page_number+1): + DutuSpider.page_url_list.append(self.spider_url + str(page)) + + def __set_header(self): + """ + 定义header + :return: + """ + header = { + 'Upgrade-Insecure-Requests': '1', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36', + } + return header + + def __add_urllist(self): + """ + 定义从page_url_list 爬取具体的image的url + :return: + """ + while True: + DutuSpider.rlock.acquire() + if len(DutuSpider.page_url_list) == 0: + DutuSpider.rlock.release() + break + else: + page_url = DutuSpider.page_url_list.pop() + DutuSpider.rlock.release() + response = requests.get(page_url, headers=self.__set_header()) + soup = BeautifulSoup(response.content,'lxml') + sou_list = soup.find_all('img',attrs={'class':'img-responsive lazy image_dta'}) + # 将获取到的具体表情图标的url保存添加进img_url_list 列表 + for url_content in sou_list: + DutuSpider.rlock.acquire() + DutuSpider.img_url_list.append(url_content['data-original']) + DutuSpider.rlock.release() + + + + def __download_img(self): + """ + 从image_url_list中来下载image到本地 + :return: + """ + while True: + DutuSpider.rlock.acquire() + if len(DutuSpider.img_url_list) == 0: + DutuSpider.rlock.release() + continue + else: + img_url = DutuSpider.img_url_list.pop() + DutuSpider.rlock.release() + try: + # 图片名称 + img_name = img_url.split('/')[-1] + # 下载图片 + urllib.urlretrieve(img_url,os.path.join(self.img_dir,img_name)) + print('donload img %s' % img_name) + except Exception as e: + pass + + + def run(self): + # 启动thread_num个进程来爬去具体的img url 链接 + for th in range(self.thread_num): + add_pic_t = threading.Thread(target=self.__add_urllist) + add_pic_t.start() + + # 启动thread_num个来下载图片 + for img_th in range(self.thread_num): + download_t = threading.Thread(target=self.__download_img) + download_t.start() + + + +if __name__ == '__main__': + spider = DutuSpider(page_number=10,img_dir='imgdir',thread_number=7) + spider.get_url() + spider.run() + + diff --git a/doutulai/scrapy_doutulai/scrapy.cfg b/doutulai/scrapy_doutulai/scrapy.cfg new file mode 100644 index 0000000..5f219c5 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = scrapy_doutulai.settings + +[deploy] +#url = http://localhost:6800/ +project = scrapy_doutulai diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/__init__.py b/doutulai/scrapy_doutulai/scrapy_doutulai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/items.py b/doutulai/scrapy_doutulai/scrapy_doutulai/items.py new file mode 100644 index 0000000..9a24a48 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/items.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class ScrapyDoutulaiItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + # 定义图片url和name + img_url = scrapy.Field() + img_name = scrapy.Field() + diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/main.py b/doutulai/scrapy_doutulai/scrapy_doutulai/main.py new file mode 100644 index 0000000..7e82bf1 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/main.py @@ -0,0 +1,8 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +from scrapy import cmdline +# 执行函数 +cmdline.execute('scrapy crawl doutulai_spider'.split()) + diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py b/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py new file mode 100644 index 0000000..22d240e --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/middlewares.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class ScrapyDoutulaiSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class ScrapyDoutulaiDownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py b/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py new file mode 100644 index 0000000..98cfd04 --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/pipelines.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +import os +from urllib import urlretrieve +from scrapy_doutulai.settings import DOWNLOAD_DIR + +class ScrapyDoutulaiPipeline(object): + def __init__(self): + """ + 判断下载目录是否存在 + """ + if not os.path.exists(DOWNLOAD_DIR): + os.makedirs(DOWNLOAD_DIR) + + def process_item(self, item, spider): + """ + 下载图片 + :param item: + :param spider: + :return: + """ + try: + filename = os.path.join(DOWNLOAD_DIR,item['img_name']) + print(filename) + urlretrieve(item['img_url'],filename) + except Exception as e: + pass \ No newline at end of file diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py b/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py new file mode 100644 index 0000000..215e3fc --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/settings.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for scrapy_doutulai project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'scrapy_doutulai' + +SPIDER_MODULES = ['scrapy_doutulai.spiders'] +NEWSPIDER_MODULE = 'scrapy_doutulai.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'scrapy_doutulai (+http://www.yourdomain.com)' +USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'scrapy_doutulai.middlewares.ScrapyDoutulaiSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'scrapy_doutulai.middlewares.ScrapyDoutulaiDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://doc.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'scrapy_doutulai.pipelines.ScrapyDoutulaiPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +# 定义图片下载目录 +DOWNLOAD_DIR = 'imagedir' \ No newline at end of file diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc b/doutulai/scrapy_doutulai/scrapy_doutulai/spiders/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e893c84a71652bb876e8b16ce6201b22c6bd46f GIT binary patch literal 180 zcmZSn%*&-+TpgXv00oRd+5w1*S%5?e14FO|NW@PANHCxg#j!v!7ps_p%94!yyz=~_ z?3jQ`5LH}~Sdy8a7jyK$=4UY}`K2YLIf Date: Mon, 27 Aug 2018 16:38:16 +0800 Subject: [PATCH 18/21] add file --- README.md | 5 +++++ cvmoper/{cmv_oper.py => cvm_oper.py} | 0 cvmoper/cvmlog/2018-08-27-cvm-oper.log | 6 ++++++ 3 files changed, 11 insertions(+) rename cvmoper/{cmv_oper.py => cvm_oper.py} (100%) create mode 100644 cvmoper/cvmlog/2018-08-27-cvm-oper.log diff --git a/README.md b/README.md index 6b40b2e..75d943a 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,8 @@ blog：[Python实现ECS自动镜像创建&镜像复制至其他地域](http://bl > Python多线程爬图&Scrapy框架爬图 blog：[Python多线程爬图&Scrapy框架爬图](http://blog.51cto.com/kaliarch/2162411) + +## Python操作CVM(cvmoper) +> Python操作CVM + +blog：[Python操作CVM](http://blog.51cto.com/kaliarch/2165000) diff --git a/cvmoper/cmv_oper.py b/cvmoper/cvm_oper.py similarity index 100% rename from cvmoper/cmv_oper.py rename to cvmoper/cvm_oper.py diff --git a/cvmoper/cvmlog/2018-08-27-cvm-oper.log b/cvmoper/cvmlog/2018-08-27-cvm-oper.log new file mode 100644 index 0000000..b2a67a7 --- /dev/null +++ b/cvmoper/cvmlog/2018-08-27-cvm-oper.log @@ -0,0 +1,6 @@ +2018-08-27 16:35:34,533 - root - INFO - ------------------------start cvm of API log------------- +2018-08-27 16:35:35,092 - root - INFO - cvm [u'ins-h8d3e69v'] reboot successful! +2018-08-27 16:35:35,092 - root - INFO - {"RequestId": "17840c5f-65a9-41a1-9ab6-9e03e55d286b"} +2018-08-27 16:37:25,822 - root - INFO - ------------------------start cvm of API log------------- +2018-08-27 16:37:26,433 - root - INFO - cvm [u'ins-h8d3e69v'] reboot successful! +2018-08-27 16:37:26,434 - root - INFO - {"RequestId": "fe987e97-a80d-4a9a-ba6b-19b88b069d93"} From dc045b414b2c4113d01483af22d077ec09b491f4 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Mon, 27 Aug 2018 16:55:49 +0800 Subject: [PATCH 19/21] change fiel --- cvmoper/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvmoper/config.py b/cvmoper/config.py index 57491dd..4f8bc60 100644 --- a/cvmoper/config.py +++ b/cvmoper/config.py @@ -7,7 +7,7 @@ SecretId = AKIDjPYbTBU4FF4xxxxxxxxxxxxxxxxxxxxxx # 腾讯云secretkey SecretKey = e7RaXYVP63rUvBNUQF1iKxxxxxxxx -# cvm 所在地域 +# cvm 所在地域,可参考https://cloud.tencent.com/document/api/213/15708 Region = ap-shanghai # 腾讯云cvm实例id，多个用,隔开 From 2058ab3b3a4b025b35d2c9230798e7636bb187f6 Mon Sep 17 00:00:00 2001 From: KaliArch <18329903316@163.com> Date: Fri, 31 Aug 2018 13:14:09 +0800 Subject: [PATCH 20/21] Update config.cfg --- imageoper/config.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imageoper/config.cfg b/imageoper/config.cfg index 3270493..56828a6 100644 --- a/imageoper/config.cfg +++ b/imageoper/config.cfg @@ -1,9 +1,9 @@ # 阿里云ak配置，建议采用子账户只授权ecs镜像操作 [common] # 阿里云acccesskeyid -accessKeyId = LTAIhfXlcjyln6tW +accessKeyId = LTAIhfxxxxxxxx # 阿里云accesssecret -accessSecret = GwfAMvR4K2ELmt76184oqLTVgRfAso +accessSecret = GwfAMvR4Kxxxxxxxxxxxxxxxxxxx # log目录名称 logdir_name = logdir # log文件名称 From 680de6818be172aba4cc56f3f23ff7779dc65222 Mon Sep 17 00:00:00 2001 From: redhatxl <18329903316@163.com> Date: Wed, 17 Oct 2018 17:25:19 +0800 Subject: [PATCH 21/21] add sa_blog --- README.md | 5 ++ save_blog/save_blog.py | 101 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 save_blog/save_blog.py diff --git a/README.md b/README.md index 75d943a..6715d4a 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,8 @@ blog：[Python多线程爬图&Scrapy框架爬图](http://blog.51cto.com/kaliarch > Python操作CVM blog：[Python操作CVM](http://blog.51cto.com/kaliarch/2165000) + +## 利用Python批量保存51CTO博客 +> 利用Python批量保存51CTO博客 + +blog：[利用Python批量保存51CTO博客](http://blog.51cto.com/kaliarch/2301359) diff --git a/save_blog/save_blog.py b/save_blog/save_blog.py new file mode 100644 index 0000000..3290252 --- /dev/null +++ b/save_blog/save_blog.py @@ -0,0 +1,101 @@ +#!/bin/env python +# -*- coding:utf-8 -*- +# _auth:kaliarch + +import requests +import time +from bs4 import BeautifulSoup +from selenium import webdriver + + +class BlogSave(): + # 定义headers字段 + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36" + } + + def __init__(self,blog_name,page_number,login_user_name,login_passwd): + self.login_url = 'http://home.51cto.com/index' + # 博客用户名 + self.blog_name = blog_name + # 需要保存的博客多少页 + self.page_number = page_number + # 登陆的用户 + self.login_user_name = login_user_name + # 登陆的密码 + self.login_passwd = login_passwd + # 本地的chreomedriver驱动 + self.chromedirve = 'D:\chromedriver.exe' + # blog 导入url + self.blog_save_url = 'http://blog.51cto.com/blogger/publish/' + + + def get_urldict(self): + """ + 爬去用户文章的url + :param pagenumber: + :return: urllist + """ + content_dict = {} + scrapy_urllist = ["http://blog.51cto.com/" + str(self.blog_name) + "/p" + str(page) for page in + range(1, int(self.page_number) + 1)] + for scrapy_url in scrapy_urllist: + response = requests.get(scrapy_url, headers=BlogSave.headers) + soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf-8') + title_list = soup.find_all('a', class_='tit') + + for content in title_list: + # 获取url + url = content['href'] + title_soup = BeautifulSoup(requests.get(url, headers=BlogSave.headers).content, 'lxml', from_encoding='utf-8') + title = title_soup.find_all('h1', class_='artical-title') + # 获取标题 + # print(title[0].get_text()) + content_dict[title[0].get_text()] = url + print(title[0].get_text(),url) + + return content_dict + + + def save_blog(self,url_list): + """ + 通过模拟登陆保存博客文件 + :return: + """ + browser = webdriver.Chrome(self.chromedirve) + # 打开url + browser.get(self.login_url) + time.sleep(2) + # 登陆 + browser.find_element_by_id('loginform-username').send_keys(self.login_user_name) + browser.find_element_by_id('loginform-password').send_keys(self.login_passwd) + browser.find_element_by_name('login-button').click() + time.sleep(1) + for url in url_list: + browser.get(url) + time.sleep(1) + try: + browser.find_element_by_xpath('//*[@id="blogEditor-box"]/div[1]/a[14]').click() + time.sleep(2) + except Exception as e: + with open('fail.log','a') as f: + f.write(url + str(e)) + + def run(self): + # 获取标题和url字典 + content_dict = self.get_urldict() + # 获取url列表 + id_list = [] + for value in content_dict.values(): + id_list.append(str(value).split('/')[-1]) + result_list = [ self.blog_save_url + str(id) for id in id_list ] + print("result_list:",result_list) + self.save_blog(result_list) + +if __name__ == '__main__': + # blogOper = BlogSave('kaliarch',1) + # dict = blogOper.get_urldict() + # value_list = [ value for value in dict.values()] + # print(value_list) + blogOper = BlogSave(blog_name='kaliarch',page_number=5,login_user_name='xxxxxxxxxxxxx@163.com',login_passwd='qxxxxxxxxx') + blogOper.run()