diff --git a/weibo.py b/weibo.py index db05c0f..eb37416 100755 --- a/weibo.py +++ b/weibo.py @@ -711,47 +711,25 @@ def get_one_page(self, page): js = self.get_weibo_json(page) if js['ok']: weibos = js['data']['cards'] - if self.query: - weibos = weibos[0]['card_group'] for w in weibos: if w['card_type'] == 9: - wb = self.get_one_weibo(w) - if wb: - if wb['id'] in self.weibo_id_list: - continue - created_at = datetime.strptime( - wb['created_at'], '%Y-%m-%d') - since_date = datetime.strptime( - self.user_config['since_date'], '%Y-%m-%d') - if created_at < since_date: - if self.is_pinned_weibo(w): - continue - else: - logger.info( - u'{}已获取{}({})的第{}页{}微博{}'.format( - '-' * 30, self.user['screen_name'], - self.user['id'], page, - '包含"' + self.query + - '"的' if self.query else '', - '-' * 30)) - return True - if (not self.is_pinned_weibo(w)) \ - and ((not self.filter) - or ('retweet' not in wb.keys())): - self.weibo.append(wb) - self.weibo_id_list.append(wb['id']) - self.got_count += 1 - self.print_weibo(wb) - else: - logger.info(u'正在过滤转发微博') - else: - return True - logger.info(u'{}已获取{}({})的第{}页微博{}'.format( - '-' * 30, self.user['screen_name'], self.user['id'], page, - '-' * 30)) + wb = w['mblog'] + if (not self.filter) or ('retweeted_status' not in wb): + self.weibo.append(self.parse_weibo(wb)) + if self.user['statuses_count'] > 0: + progress = self.got_count / self.user['statuses_count'] + progress = '%.1f' % (progress * 100) + '%' + else: + progress = '100%' + logger.info(u'已获取%s的%s的微博%s条', self.user['screen_name'], + progress, self.got_count) except Exception as e: logger.exception(e) + if self.got_count >= self.user['statuses_count']: + return True + return False + def get_page_count(self): """获取微博页数""" try: diff --git a/xpost.py b/xpost.py index e9e3674..8567762 100755 --- a/xpost.py +++ b/xpost.py @@ -52,6 +52,13 @@ logger = logging.getLogger('xpost') ### Mastodon +def is_valid_url(url): + """Check if a URL is valid.""" + try: + result = requests.utils.urlparse(url) + return all([result.scheme, result.netloc]) + except ValueError: + return False def access_token(): """Return the access token for mastodon app.""" @@ -85,7 +92,7 @@ def collect_media_url(post, recursive=False): return url_list -def upload_media(url_list, max_attatchment): +def upload_media(url_list, max_attatchment, mast): """Upload media in URL_LIST. URL_LIST should be a list of MEDIA_URL. Return (TOOT_LIST, TOO_LARGE, TOO_MANY). TOOT_LIST is a list of TOOT_DICT. @@ -98,6 +105,9 @@ def upload_media(url_list, max_attatchment): media_too_many = True for media_url in url_list: url = media_url['url'] + if not is_valid_url(url): + logger.warning(f'Invalid URL: {url}') + continue resp = requests.get(url) mime = resp.headers['content-type'].split(';')[0].strip() # https://mastodonpy.readthedocs.io/en/stable/#media-post @@ -110,13 +120,12 @@ def upload_media(url_list, max_attatchment): logger.warning(f'Problem uploading media, type: {mime}, url: {url}, error: {err}') return (media_list, media_too_large, media_too_many) - def cross_post(post, mast_dict, config, db, fallback_mast=None): """Cross-post POST to mastodon. -MAST_DICT is a hash map from weibo author ids (string) to Mastodon -instances. Return a list of POST_RECORD. FALLBACK_MAST is used when we -cannot find a Mastodon instance from dict for the weibo author. -""" + MAST_DICT is a hash map from weibo author ids (string) to Mastodon + instances. Return a list of POST_RECORD. FALLBACK_MAST is used when we + cannot find a Mastodon instance from dict for the weibo author. + """ if not should_cross_post(post, config, db): return [] @@ -133,22 +142,21 @@ def cross_post(post, mast_dict, config, db, fallback_mast=None): post_record_list = [] orig_toot_id = None - # Maybe upload media. + # Maybe upload media. instance to toot with') url_list = collect_media_url(post, not standalone_repost) media_list = None media_too_large = False media_too_many = False - if not external_media: - media_list, media_too_large, media_too_many = \ - upload_media(url_list, max_attatchment) - # Come up with a Mastodon instance for tooting. mast = mast_dict.get(user_id) - if mast == None: - if fallback_mast != None: + if mast is None: + if fallback_mast is not None: mast = fallback_mast else: raise KeyError('Couldn\'t find a Mastodon instance to toot with') + if not external_media: + media_list, media_too_large, media_too_many = \ + upload_media(url_list, max_attatchment, mast) # Compose toot. # 1. Compose body text. @@ -221,6 +229,7 @@ def cross_post(post, mast_dict, config, db, fallback_mast=None): media_ids=media_list) post_record_list.append(make_post_record(post, toot)) return post_record_list + def delete_all_toots(mast):