Skip to content

fix(scraper): add request timeout and HTTP error handling to prevent workflow hangs #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
glatinone wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(scraper): add request timeout and HTTP error handling to prevent workflow hangs #10

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
503204b
fix(scraper): add request timeout and handle http errors
glatinone Jul 8, 2026
72da163
fix(scraper): add request timeout and handle http errors
glatinone Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
260 changes: 130 additions & 130 deletions scraper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,130 +1,130 @@
# coding:utf-8
import os
import datetime
import requests
import urllib.parse
from pyquery import PyQuery as pq
def scrape_url(url):
''' Scrape github trending url
'''
HEADERS = {
'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding' : 'gzip,deflate,sdch',
'Accept-Language' : 'zh-CN,zh;q=0.8'
}
print(url)
r = requests.get(url, headers=HEADERS)
assert r.status_code == 200
d = pq(r.content)
items = d('div.Box article.Box-row')
results = {}
# codecs to solve the problem utf-8 codec like chinese
for item in items:
i = pq(item)
title = i(".lh-condensed a").text()
description = i("p.col-9").text()
url = i(".lh-condensed a").attr("href")
url = "https://github.com" + url
results[title] = { 'title': title, 'url': url, 'description': description }
return results
def scrape_lang(language):
''' Scrape github trending with lang parameters
'''
url = 'https://github.com/trending/{language}'.format(language=urllib.parse.quote_plus(language))
r1 = scrape_url(url)
url = 'https://github.com/trending/{language}?spoken_language_code=zh'.format(language=urllib.parse.quote_plus(language))
r2 = scrape_url(url)
return { **r1, **r2 }
def write_markdown(lang, results, archived_contents):
''' Write the results to markdown file
'''
content = ''
with open('README.md', mode='r', encoding='utf-8') as f:
content = f.read()
content = convert_file_contenet(content, lang, results, archived_contents)
with open('README.md', mode='w', encoding='utf-8') as f:
f.write(content)
def is_title_exist(title, content, archived_contents):
if '[' + title + ']' in content:
return True
for archived_content in archived_contents:
if '[' + title + ']' in archived_content:
return True
return False
def convert_file_contenet(content, lang, results, archived_contents):
''' Add distinct results to content
'''
distinct_results = []
for title, result in results.items():
if not is_title_exist(title, content, archived_contents):
distinct_results.append(result)
if not distinct_results:
print('There is no distinct results')
return content
lang_title = convert_lang_title(lang)
if lang_title not in content:
content = content + lang_title + '\n\n'
return content.replace(lang_title + '\n\n', lang_title + '\n\n' + convert_result_content(distinct_results))
def convert_result_content(results):
''' Format all results to a string
'''
strdate = datetime.datetime.now().strftime('%Y-%m-%d')
content = ''
for result in results:
content = content + u"* 【{strdate}】[{title}]({url}) - {description}\n".format(
strdate=strdate, title=result['title'], url=result['url'],
description=format_description(result['description']))
return content
def format_description(description):
''' Remove new line characters
'''
if not description:
return ''
return description.replace('\r', '').replace('\n', '')
def convert_lang_title(lang):
''' Lang title
'''
if lang == '':
return '## All language'
return '## ' + lang.capitalize()
def get_archived_contents():
archived_contents = []
archived_files = os.listdir('./archived')
for file in archived_files:
content = ''
with open('./archived/' + file, mode='r', encoding='utf-8') as f:
content = f.read()
archived_contents.append(content)
return archived_contents
def job():
''' Get archived contents
'''
archived_contents = get_archived_contents()
''' Start the scrape job
'''
languages = ['', 'java', 'python', 'javascript', 'go', 'c', 'c++', 'c#', 'html', 'css', 'unknown']
for lang in languages:
results = scrape_lang(lang)
write_markdown(lang, results, archived_contents)
if __name__ == '__main__':
job()
# coding:utf-8

import os
import datetime
import requests
import urllib.parse
from pyquery import PyQuery as pq
from requests import exceptions as req_exc

# Default request timeout (seconds), configurable via env
REQUEST_TIMEOUT = float(os.environ.get('REQUEST_TIMEOUT', '15'))

def scrape_url(url):
''' Scrape github trending url '''
HEADERS = {
'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding' : 'gzip,deflate,sdch',
'Accept-Language' : 'zh-CN,zh;q=0.8'
}

print(url)
try:
r = requests.get(url, headers=HEADERS, timeout=REQUEST_TIMEOUT)
r.raise_for_status()
d = pq(r.content)
items = d('div.Box article.Box-row')

results = {}
# codecs to solve the problem utf-8 codec like chinese
for item in items:
i = pq(item)
title = i(".lh-condensed a").text()
description = i("p.col-9").text()
href = i(".lh-condensed a").attr("href")
href = "https://github.com" + href if href else None
if title:
results[title] = { 'title': title, 'url': href, 'description': description }
return results
except (req_exc.Timeout, req_exc.RequestException) as e:
# Log error and return empty dict to allow caller to continue gracefully
print("scrape_url error for {}: {}".format(url, e))
return {}

def scrape_lang(language):
''' Scrape github trending with lang parameters '''
url = 'https://github.com/trending/{language}'.format(language=urllib.parse.quote_plus(language))
r1 = scrape_url(url)
url = 'https://github.com/trending/{language}?spoken_language_code=zh'.format(language=urllib.parse.quote_plus(language))
r2 = scrape_url(url)
return { **r1, **r2 }

def write_markdown(lang, results, archived_contents):
''' Write the results to markdown file '''
content = ''
with open('README.md', mode='r', encoding='utf-8') as f:
content = f.read()
content = convert_file_contenet(content, lang, results, archived_contents)
with open('README.md', mode='w', encoding='utf-8') as f:
f.write(content)

def is_title_exist(title, content, archived_contents):
if '[' + title + ']' in content:
return True
for archived_content in archived_contents:
if '[' + title + ']' in archived_content:
return True
return False

def convert_file_contenet(content, lang, results, archived_contents):
''' Add distinct results to content '''
distinct_results = []
for title, result in results.items():
if not is_title_exist(title, content, archived_contents):
distinct_results.append(result)

if not distinct_results:
print('There is no distinct results')
return content

lang_title = convert_lang_title(lang)
if lang_title not in content:
content = content + lang_title + '\n\n'

return content.replace(lang_title + '\n\n', lang_title + '\n\n' + convert_result_content(distinct_results))

def convert_result_content(results):
''' Format all results to a string '''
strdate = datetime.datetime.now().strftime('%Y-%m-%d')
content = ''
for result in results:
content = content + u"* 【{strdate}】[{title}]({url}) - {description}\n".format(
strdate=strdate, title=result['title'], url=result['url'],
description=format_description(result['description']))
return content

def format_description(description):
''' Remove new line characters '''
if not description:
return ''
return description.replace('\r', '').replace('\n', '')

def convert_lang_title(lang):
''' Lang title '''
if lang == '':
return '## All language'
return '## ' + lang.capitalize()

def get_archived_contents():
archived_contents = []
archived_files = os.listdir('./archived')
for file in archived_files:
content = ''
with open('./archived/' + file, mode='r', encoding='utf-8') as f:
content = f.read()
archived_contents.append(content)
return archived_contents

def job():
''' Get archived contents '''
archived_contents = get_archived_contents()

''' Start the scrape job '''
languages = ['', 'java', 'python', 'javascript', 'go', 'c', 'c++', 'c#', 'html', 'css', 'unknown']
for lang in languages:
results = scrape_lang(lang)
write_markdown(lang, results, archived_contents)

if __name__ == '__main__':
job()