From 35d9b06e872cb35858afb9039a7b30400590d990 Mon Sep 17 00:00:00 2001 From: kartofen Date: Sun, 27 Nov 2022 19:05:00 +0200 Subject: save to file in inf loop --- scraper.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'scraper.py') diff --git a/scraper.py b/scraper.py index a7a5c50..336787d 100755 --- a/scraper.py +++ b/scraper.py @@ -1,7 +1,9 @@ #!/usr/bin/python3 +import sys import time import json import requests + from datetime import datetime from datetime import timedelta @@ -14,6 +16,10 @@ from bs4 import BeautifulSoup url = 'https://www.foxsports.com/soccer/2022-fifa-world-cup/scores?date=2022-11-20' year = '2022' +if len(sys.argv) != 3: + print("You need to provide 2 arguments: filename and time between scrapes") + exit(1) + def day_at(when): date = datetime.now() if when == "Yesterday": @@ -63,9 +69,9 @@ options = Options() options.headless = True driver = webdriver.Chrome(options=options, executable_path='chromedriver') -def scrape(): +def scrape(filename = ""): driver.get(url) - time.sleep(5) + time.sleep(6) soup = BeautifulSoup(driver.page_source, "lxml") days = [] @@ -83,6 +89,8 @@ def scrape(): days.append(Day(date, matches)) - write_as_json("", days, 2) + write_as_json(filename, days, 2) -scrape() +while 1: + scrape(sys.argv[1]) + time.sleep(int(sys.argv[2])) -- cgit v1.2.3