Update xpost.py

weibo.py

@@ -711,47 +711,25 @@ def get_one_page(self, page):
             js = self.get_weibo_json(page)
             if js['ok']:
                 weibos = js['data']['cards']
-                if self.query:
-                    weibos = weibos[0]['card_group']
                 for w in weibos:
                     if w['card_type'] == 9:
-                        wb = self.get_one_weibo(w)
-                        if wb:
-                            if wb['id'] in self.weibo_id_list:
-                                continue
-                            created_at = datetime.strptime(
-                                wb['created_at'], '%Y-%m-%d')
-                            since_date = datetime.strptime(
-                                self.user_config['since_date'], '%Y-%m-%d')
-                            if created_at < since_date:
-                                if self.is_pinned_weibo(w):
-                                    continue
-                                else:
-                                    logger.info(
-                                        u'{}已获取{}({})的第{}页{}微博{}'.format(
-                                            '-' * 30, self.user['screen_name'],
-                                            self.user['id'], page,
-                                            '包含"' + self.query +
-                                            '"的' if self.query else '',
-                                            '-' * 30))
-                                    return True
-                            if (not self.is_pinned_weibo(w)) \
-                               and ((not self.filter)
-                                    or ('retweet' not in wb.keys())):
-                                self.weibo.append(wb)
-                                self.weibo_id_list.append(wb['id'])
-                                self.got_count += 1
-                                self.print_weibo(wb)
-                            else:
-                                logger.info(u'正在过滤转发微博')
-            else:
-                return True
-            logger.info(u'{}已获取{}({})的第{}页微博{}'.format(
-                '-' * 30, self.user['screen_name'], self.user['id'], page,
-                '-' * 30))
+                        wb = w['mblog']
+                        if (not self.filter) or ('retweeted_status' not in wb):
+                            self.weibo.append(self.parse_weibo(wb))
+                if self.user['statuses_count'] > 0:
+                    progress = self.got_count / self.user['statuses_count']
+                    progress = '%.1f' % (progress * 100) + '%'
+                else:
+                    progress = '100%'
+                logger.info(u'已获取%s的%s的微博%s条', self.user['screen_name'],
+                            progress, self.got_count)
         except Exception as e:
             logger.exception(e)
 
+        if self.got_count >= self.user['statuses_count']:
+            return True
+        return False
+
     def get_page_count(self):
         """获取微博页数"""
         try:

xpost.py

@@ -52,6 +52,13 @@
 logger = logging.getLogger('xpost')
 
 ### Mastodon
+def is_valid_url(url):
+    """Check if a URL is valid."""
+    try:
+        result = requests.utils.urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
 
 def access_token():
     """Return the access token for mastodon app."""
@@ -85,7 +92,7 @@ def collect_media_url(post, recursive=False):
     return url_list
 
 
-def upload_media(url_list, max_attatchment):
+def upload_media(url_list, max_attatchment, mast):
     """Upload media in URL_LIST.
 URL_LIST should be a list of MEDIA_URL. Return (TOOT_LIST, TOO_LARGE,
 TOO_MANY). TOOT_LIST is a list of TOOT_DICT.
@@ -98,6 +105,9 @@ def upload_media(url_list, max_attatchment):
         media_too_many = True
     for media_url in url_list:
         url = media_url['url']
+        if not is_valid_url(url):
+            logger.warning(f'Invalid URL: {url}')
+            continue
         resp = requests.get(url)
         mime = resp.headers['content-type'].split(';')[0].strip()
         # https://mastodonpy.readthedocs.io/en/stable/#media-post
@@ -110,13 +120,12 @@ def upload_media(url_list, max_attatchment):
                 logger.warning(f'Problem uploading media, type: {mime}, url: {url}, error: {err}')
     return (media_list, media_too_large, media_too_many)
 
-
 def cross_post(post, mast_dict, config, db, fallback_mast=None):
     """Cross-post POST to mastodon.
-MAST_DICT is a hash map from weibo author ids (string) to Mastodon
-instances. Return a list of POST_RECORD. FALLBACK_MAST is used when we
-cannot find a Mastodon instance from dict for the weibo author.
-"""
+    MAST_DICT is a hash map from weibo author ids (string) to Mastodon
+    instances. Return a list of POST_RECORD. FALLBACK_MAST is used when we
+    cannot find a Mastodon instance from dict for the weibo author.
+    """
     if not should_cross_post(post, config, db):
         return []
 
@@ -133,22 +142,21 @@ def cross_post(post, mast_dict, config, db, fallback_mast=None):
     post_record_list = []
     orig_toot_id = None
 
-    # Maybe upload media.
+    # Maybe upload media. instance to toot with')
     url_list = collect_media_url(post, not standalone_repost)
     media_list = None
     media_too_large = False
     media_too_many = False
-    if not external_media:
-        media_list, media_too_large, media_too_many = \
-            upload_media(url_list, max_attatchment)
-
     # Come up with a Mastodon instance for tooting.
     mast = mast_dict.get(user_id)
-    if mast == None:
-        if fallback_mast != None:
+    if mast is None:
+        if fallback_mast is not None:
             mast = fallback_mast
         else:
             raise KeyError('Couldn\'t find a Mastodon instance to toot with')
+    if not external_media:
+        media_list, media_too_large, media_too_many = \
+            upload_media(url_list, max_attatchment, mast)
 
     # Compose toot.
     # 1. Compose body text.
@@ -221,6 +229,7 @@ def cross_post(post, mast_dict, config, db, fallback_mast=None):
                             media_ids=media_list)
     post_record_list.append(make_post_record(post, toot))
     return post_record_list
+
 
 
 def delete_all_toots(mast):