Racenet implemented

_bs.py

@@ -6,7 +6,7 @@ import model
 import collections
 from string import capwords
 # import pytz
-# from pprint import pprint
+from pprint import pprint
 
 """
 This module contains custom methods based on bs4.beautifulsoup to analyze data
@@ -200,3 +200,18 @@ def process_scratchings(this_data, this_venue):
     #        len(scratchings), scratchings_count))
     #    pprint(scratchings)
     return scratchings
+
+
+def get_racenet_json(html):
+    this_soup = BeautifulSoup(html, 'html.parser')
+    pattern = re.compile(r'window\.initialReduxState = (.*)')
+    script = this_soup.find('script', text=pattern)
+    json = '{}'
+    if script:
+        print('script')
+        match = pattern.search(script.text)
+        if match:
+            print('match')
+            json = match.group(1)
+    pprint(json)
+    return json

_html.py

@@ -13,5 +13,6 @@ def get_page(this_url):
     :param this_url:
     :return:
     """
-    data = requests.get(this_url)
+    headers = {'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
+    data = requests.get(this_url, headers=headers)
     return data.text

main.py

@@ -24,7 +24,7 @@ if __name__ == '__main__':
 
     db = database.db
 
-    race_days_global = model.scrape_main_page(row)
+    race_days_global = model.scrape_racingaustralia_main_page(row)
     interim = time.time()
     # print('interim 1 {}'.format(interim - start))
     # pprint(race_days_global)

model.py

@@ -12,8 +12,10 @@ import view
 
 
 import warnings
-
-warnings.simplefilter("ignore", arrow.factory.ArrowParseWarning)
+try:
+    warnings.simplefilter("ignore", arrow.factory.ArrowParseWarning)
+except AttributeError:
+    pass
 
 """
 Modules _html and _bs4 contain specialized methods.
@@ -30,13 +32,21 @@ local_timezones = {
     "NT": "Australia/Darwin"}
 
 
-def scrape_main_page(row):
+def scrape_racingaustralia_main_page(row):
     this_url = """https://racingaustralia.horse/Home.aspx"""
     this_data = _html.get_page(this_url)
     venues_all = _bs.get_today_row(this_data, row)
     return venues_all
 
 
+def scrape_racenet_main_page():
+    this_url = """https://www.racenet.com.au/updates/scratchings"""
+    this_data = _html.get_page(this_url)
+    print(this_data[:50])
+    json = _bs.get_racenet_json(this_data)
+    return json
+
+
 def get_raw_scratchings(this_venue):
     this_raw_data = _html.get_page(this_venue.scratchings_url)
     return this_raw_data
@@ -99,13 +109,13 @@ def send_messages(scratches, source):
             flag = 'FLAGGED!! '
         message = message_string.format(source,
                                         flag,
-                                        m.date.format('%a'),
+                                        m.date.strftime('%a'),
                                         m.date,
                                         m.state,
                                         m.venue,
                                         m.race,
-                                        m.time.strftime('%H:%M'),
-                                        m.utc.strftime('%H:%M'),
+                                        m.time,
+                                        m.utc,
                                         '{} {}'.format(m.horse_no, m.horse_display_name))
         print('this_message: {}'.format(message))
         # Append message if possible

racenet.py

@@ -2,14 +2,24 @@ import json
 # from pprint import pprint
 from model import local_timezones
 import arrow
-import collections
+# import collections
+import _bs
+import model
+# import re
+import textwrap
+import database
 
-Data = collections.namedtuple('Data', 'venue state race time horse_no horse flag')
+# Data = collections.namedtuple('Data', 'venue state race time horse_no horse flag')
 
-with open('scratchings.json', 'r') as f:
-    data_all = json.loads(f.read())
-    data = data_all["MainPageState"]["receivedData"][0]
-    name = data['name']
+my_json = model.scrape_racenet_main_page()
+my_json = textwrap.fill(my_json[:-1], 1e6)
+# print(my_json)
+# exit(0)
+# with open('scratchings.json', 'r') as f:
+data_all = json.loads(my_json)
+# pprint(data_all)
+data = data_all["MainPageState"]["receivedData"][0]
+name = data['name']
 
 print(name)
 items = 0
@@ -25,6 +35,7 @@ for d in data['data']:
                                 timezone)
     print(venue)
     races = d['races']
+    scratchings = []
     for race in races:
         raceNumber = race['raceNumber']
         startTimeLocal = race['startTimeLocal'][11:-3]
@@ -40,7 +51,16 @@ for d in data['data']:
                 horseDisplayName = runner['horseDisplayName']
                 horseNumber = runner['horseNumber']
                 # print('\t{:2d}: {}'.format(horseNumber, horseDisplayName))
-                data_without_flag = Data(d['venue']['name'], d['venue']['state'] if is_aus else 'NZL',
-                                         raceNumber, arrowStartTimeLocal,
-                                         horseNumber, horseDisplayName, False)
-                print(data_without_flag)
+                # data_without_flag = Data(d['venue']['name'], d['venue']['state'] if is_aus else 'NZL',
+                #                         raceNumber, arrowStartTimeLocal,
+                #                         horseNumber, horseDisplayName, False)
+                data_without_flag = _bs.Scratching(d['venue']['name'], d['venue']['state'] if is_aus else 'NZL',
+                                                   arrowStartTimeLocal.date(), raceNumber,
+                                                   arrowStartTimeLocal.format('HH:mm'),
+                                                   arrowStartTimeLocal.to('utc').format('HH:mm'), horseNumber,
+                                                   horseDisplayName, False)
+                scratchings.append(data_without_flag)
+    db = database.db
+    scratchings_to_be_broadcast = model.store_scratched_horses(db, scratchings)
+    # print('len(scratchings_to_be_broadcast): {}'.format(len(scratchings_to_be_broadcast)))
+    model.send_messages(scratchings_to_be_broadcast, 'RN')