Major refactor

.idea/dictionaries/foppe.xml

@@ -6,6 +6,7 @@
       <w>beautifulsoup</w>
       <w>dotenv</w>
       <w>pytz</w>
+      <w>racenet</w>
       <w>sratchings</w>
       <w>unixtime</w>
       <w>webpage</w>

.idea/other.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PySciProjectComponent">
+    <option name="PY_SCI_VIEW" value="true" />
+  </component>
+</project>

.idea/scratchings.iml

@@ -7,6 +7,9 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="PyDocumentationSettings">
+    <option name="renderExternalDocumentation" value="true" />
+  </component>
   <component name="TestRunnerService">
     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
   </component>

_bs.py

@@ -4,6 +4,7 @@ import datetime
 from pytz import timezone
 import model
 import collections
+from string import capwords
 # import pytz
 # from pprint import pprint
 
@@ -18,7 +19,8 @@ RaceDayShort = collections.namedtuple('RaceDayShort', Venue._fields + ('date_str
 RaceDay = collections.namedtuple('RaceDay', RaceDayShort._fields + (
     'scratchings_latest_datetime', 'scratchings_latest_unixtime',
     'scratchings_close_datetime', 'scratchings_close_unixtime'))
-Scratching = collections.namedtuple('Scratching', 'venue state date race horse')
+RawScratching = collections.namedtuple('RawScratching', 'venue state date race horse_no horse_display_name')
+Scratching = collections.namedtuple('Scratching', 'venue state date race time utc horse_no horse_display_name torn')
 
 
 def get_today_row(this_text, this_row):
@@ -151,9 +153,18 @@ def scrape_scratchings(div, this_venue):
                 print('First character in line: {}'.format(text[0]))
                 print('The start of the offending line is: {}'.format(text[0:10]))
                 continue
-            text = re.sub(r'e\s+', ' ', text)
-            text = re.sub(r'\s+', ' ', text)  # Kills tabs between number and name of horse
-            temp_list = Scratching(this_venue.name, this_venue.state, this_venue.date, race, text)
+            match = re.search(r'^(\d{1,2})e?\s+(.+)', text)
+            no = 0
+            name = ''
+            if match:
+                no = int(match.group(1))
+                name = capwords(match.group(2))
+                name = re.sub(r' Of ', ' of ', name)
+                if name.endswith('(nz)'):
+                    name = name[:-len(' (nz)')]
+            # text = re.sub(r'e\s+', ' ', text)
+            # text = re.sub(r'\s+', ' ', text)  # Kills tabs between number and name of horse
+            temp_list = RawScratching(this_venue.name, this_venue.state, this_venue.date, race, no, name)
             scraped_scratchings.append(temp_list)
     return scraped_scratchings
 
@@ -172,6 +183,7 @@ def process_scratchings(this_data, this_venue):
     scratchings = set()
     early_scratchings = scrape_scratchings(div, this_venue)
     scratchings.update(early_scratchings)
+    # print('len(scratchings): {}'.format(len(scratchings)))
     header = this_scr.findAll('h3', text=re.compile('Late Scratchings'))[0]
     late_div = header.findNext('table')
     late_scratchings = scrape_scratchings(late_div, this_venue)
@@ -179,6 +191,7 @@ def process_scratchings(this_data, this_venue):
     #      pprint(late_div)
     #     pprint(late_scratchings)
     scratchings.update(late_scratchings)
+    # print('len(scratchings): {}'.format(len(scratchings)))
 
     assert len(scratchings) == int(scratchings_count), 'len(scratchings) {} == scratchings_count {}'.format(
         len(scratchings), scratchings_count)

main.py

@@ -1,12 +1,14 @@
-import datetime
+# import datetime
 
 import psycopg2.extras
 import model
+import _bs
 from pprint import pprint
 import time
 import database
 import sys
-import view
+# import view
+# import arrow
 
 if __name__ == '__main__':
     row = -1
@@ -37,8 +39,11 @@ if __name__ == '__main__':
     # print('interim 2 {}'.format(interim - start))
     # pprint(race_days)
 
+    races = model.get_relevant_races_from_database(db)
+
     scratchings_for_database = []
     messages = []
+    full_scratchings = []
     for race_day in race_days:
         if not raw_data_dict or not race_day:
             # raw_data_dict may be empty when there is no data available (yet)
@@ -61,6 +66,22 @@ if __name__ == '__main__':
 
         # compare retrieved scratchings with new data
         for scratching in scratchings:
+            additional_data = model.get_race_from_races(races, scratching.date, scratching.venue, scratching.race)
+            # print('additional_data')
+            # pprint(additional_data)
+            if len(additional_data) == 0:
+                print('len(additional_data) == {}'.format(len(additional_data)))
+                pprint(scratching)
+                print('continue')
+                continue
+            full_scratching = _bs.Scratching(scratching.venue, scratching.state, scratching.date, scratching.race,
+                                             additional_data[0], additional_data[1], scratching.horse_no,
+                                             scratching.horse_display_name, additional_data[2])
+            full_scratchings.append(full_scratching)
+            continue
+
+    """
+            # ### OLD ###
             found_in_database = False
             # if db_data is None:
             #     print(' 62. Caught db_data is None')
@@ -81,13 +102,14 @@ if __name__ == '__main__':
                             scratching.date == date_in_db_row_formatted and
                             scratching.venue == row.venue and
                             int(scratching.race) == int(row.race) and
-                            scratching.horse == row.horse
+                            scratching.horse_no == row.horse_no
                     ):
                         message_string = 'Horse found: date = {}, venue = {}, race = {}, horse = {}'
                         message = message_string.format(scratching.date,
                                                         scratching.venue,
                                                         scratching.race,
-                                                        scratching.horse)
+                                                        '{} {}'.format(
+                                                            scratching.horse_no, scratching.horse_display_name))
                         # print(message)
                         # print(row)
                         found_in_database = True
@@ -102,11 +124,11 @@ if __name__ == '__main__':
                 date_object = datetime.datetime.strptime(scratching.date, "%Y-%m-%d").date()
                 day_abbr = date_object.strftime('%a')
                 query = """
-                    SELECT start_time, utctime, torn FROM race_program 
-                    WHERE race_date = %s AND
-                        venue = %s AND
-                        race = %s;
-                """
+    #                     SELECT start_time, utctime, torn FROM race_program
+    #                     WHERE race_date = %s AND
+    #                         venue = %s AND
+    #                         race = %s;
+    """
                 cur2 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
                 cur2.execute(query, (scratching.date, scratching.venue, scratching.race))
                 db_data_query2 = cur2.fetchone()
@@ -130,17 +152,21 @@ if __name__ == '__main__':
                                                 scratching.race,
                                                 db_data_query2.start_time.strftime('%H:%M'),
                                                 db_data_query2.utctime.strftime('%H:%M'),
-                                                scratching.horse)
+                                                '{} {}'.format(scratching.horse_no, scratching.horse_display_name))
                 print(message)
                 if broadcast or flag == 'FLAGGED!! ':
                     messages.append(message)
                     # view.broadcast(message)
                 # store new scratching
                 scratchings_for_database.append((scratching.venue, scratching.date,
-                                                 scratching.race, scratching.horse))
+                                                 scratching.race, scratching.horse_no, scratching.horse_display_name))
+    """
+    pprint(full_scratchings)
+    print('len(full_scratchings): {}'.format(len(full_scratchings)))
 
-    model.send_messages(messages)
-    model.store_scratched_horses(scratchings_for_database, db)
+    scratchings_to_be_broadcast = model.store_scratched_horses(db, full_scratchings)
+    print('len(scratchings_to_be_broadcast): {}'.format(len(scratchings_to_be_broadcast)))
+    model.send_messages(scratchings_to_be_broadcast, 'RA')
 
     db.close()
 

model.py

@@ -6,6 +6,9 @@ import pytz
 import datetime
 # import time
 import psycopg2.extras
+from pprint import pprint
+import arrow
+
 
 import view
 
@@ -83,49 +86,110 @@ def convert_to_date(weird_string):
     return calculated_date
 
 
-def send_messages(messages):
+def send_messages(scratches, source):
     long_message = ''
-    leftover_message = ''
-    for this_message in messages:
-        print('this_message: {}'.format(this_message))
-        leftover_message = this_message
+
+    message_string = '{} {}venue = {} {} {}-{} | race = {} starts at {} | {} UTC | horse = {}'
+
+    for m in scratches:
+        flag = ''
+        if m.torn:
+            flag = 'FLAGGED!! '
+        message = message_string.format(source,
+                                        flag,
+                                        m.date.format('%a'),
+                                        m.date,
+                                        m.state,
+                                        m.venue,
+                                        m.race,
+                                        m.time.strftime('%H:%M'),
+                                        m.utc.strftime('%H:%M'),
+                                        '{} {}'.format(m.horse_no, m.horse_display_name))
+        print('this_message: {}'.format(message))
         # Append message if possible
-        if len(long_message) + len(this_message) < 5997:
+        if len(long_message) + len(message) < 5997:
             if len(long_message) == 0:
-                long_message = this_message
+                long_message = message
             else:
-                long_message += '\n' + this_message
-            leftover_message = ''
+                long_message += '\n' + message
         else:
             # Send long message (max 6k characters)
             print('Sending very long message > {}'.format(len(long_message)))
             view.broadcast(long_message)
             # Best would be to now store horses that were just broadcast
-            long_message = this_message
-            leftover_message = ''
+            long_message = m
 
     # Send all messages
     if len(long_message) > 0:
         print('Sending long_message > {}'.format(len(long_message)))
         view.broadcast(long_message)
-        long_message = ''
-
-    # Send only or last message
-    if len(leftover_message) > 0:
-        print('Sending leftover_message > {}'.format(len(leftover_message)))
-        view.broadcast(leftover_message)
-        leftover_message = ''
 
 
-def store_scratched_horses(horses, db):
-    query = """INSERT INTO horses(venue, race_date, race, horse) 
-    VALUES(%s, %s, %s, %s)
-    ON CONFLICT(venue, race_date, race, horse) DO NOTHING;"""
+def store_scratched_horses(db, full_scratches):
+    query = """INSERT INTO horses(venue, race_date, race, horse_no, horse_display_name) 
+    VALUES(%s, %s, %s, %s, %s)
+    ON CONFLICT(venue, race_date, race, horse_no) DO NOTHING;"""
 
+    scratches_to_return = []
+    regex = r'^INSERT \d+ (\d+)$'
     cur3 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
-    for database_entry in horses:
+    for this_scratching in full_scratches:
+        database_entry = (this_scratching.venue, this_scratching.date,
+                          this_scratching.race, this_scratching.horse_no,
+                          this_scratching.horse_display_name)
         cur3.execute(query, database_entry)
         print('Stored: {}'.format(database_entry))
         print(cur3.statusmessage)
+        match = re.match(regex, cur3.statusmessage)
+        status = 0
+        if match:
+            status = int(match.group(1))
+        if status > 0:
+            scratches_to_return.append(this_scratching)
+
     cur3.close()
     db.commit()
+    return scratches_to_return
+
+
+def get_race_from_races(this_haystack, needle_date, needle_venue, needle_race):
+    """
+    From the previous acquired database data this will return 'start_time',
+    'utctime' and torn (boolean) where the date, venue and race are given.
+    :param this_haystack:
+    :param needle_date:
+    :param needle_venue:
+    :param needle_race:
+    :return:
+    """
+    return_values = ()
+    needle_date = arrow.get(needle_date, 'YYYY-MM-DD').date()
+    for race in this_haystack:
+        # pprint(race)
+        # print(race.race_date == needle_date)
+        # print('race.race_date == needle_date: {} == {}'.format(race.race_date, needle_date))
+        # print('type(race.race_date) == type(needle_date): {} == {}'.format(type(race.race_date), type(needle_date)))
+        # print(race.venue == needle_venue)
+        # print(race.race == needle_race)
+        if ((race.race_date == needle_date) and (race.venue == needle_venue) and (
+                race.race == needle_race)):
+            return_values = (race.start_time, race.utctime, race.torn)
+            break
+    return return_values
+
+
+def get_relevant_races_from_database(db):
+    query = """
+        SELECT venue, start_time, race_date, utctime, race, torn FROM race_program 
+        WHERE race_date >= %s;
+    """
+    # Run this query once and use resulting NamedTuple for data
+    cur_races = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
+    today = arrow.utcnow().date()
+    print('today: {}'.format(today))
+    cur_races.execute(query, (today,))
+    races = cur_races.fetchall()
+    print('len(races): {}'.format(len(races)))
+    # pprint(races)
+    cur_races.close()
+    return races

racenet.py

@@ -1,6 +1,10 @@
 import json
 # from pprint import pprint
 from model import local_timezones
+import arrow
+import collections
+
+Data = collections.namedtuple('Data', 'venue state race time horse_no horse flag')
 
 with open('scratchings.json', 'r') as f:
     data_all = json.loads(f.read())
@@ -15,18 +19,28 @@ for d in data['data']:
     print('item {}'.format(items))
     country = d['venue']['countryId']
     is_aus = True if country == 'AUS' else False
+    timezone = local_timezones[d['venue']['state']] if is_aus else 'Pacific/Auckland'
     venue = '{} {}{} {}'.format(d['venue']['name'], d['venue']['countryId'],
-                                '/'+d['venue']['state'] if is_aus else '',
-                                local_timezones[d['venue']['state']] if is_aus else 'Pacific/Auckland')
+                                '/' + d['venue']['state'] if is_aus else '',
+                                timezone)
     print(venue)
     races = d['races']
     for race in races:
         raceNumber = race['raceNumber']
-        startTimeLocal = race['startTimeLocal']
+        startTimeLocal = race['startTimeLocal'][11:-3]
+        # print(race['startTimeLocal'])
+        arrowStartTimeLocal = arrow.get(race['startTimeLocal']+' '+timezone, 'YYYY-MM-DDTHH:mm:ss ZZZ')
+        # print(arrowStartTimeLocal)
         startTimeUtc = race['startTimeUtc']
-        print('race {:2d}: {} {}'.format(raceNumber, startTimeLocal, startTimeUtc))
+        race_date = race['startTimeLocal'][:10]
+        # print(race_date)
+        # print('race {:2d}: {} {}'.format(raceNumber, startTimeLocal, startTimeUtc))
         for runner in race['runners']:
             if runner['scratched']:
                 horseDisplayName = runner['horseDisplayName']
                 horseNumber = runner['horseNumber']
-                print('\t{:2d}: {}'.format(horseNumber, horseDisplayName))
+                # print('\t{:2d}: {}'.format(horseNumber, horseDisplayName))
+                data_without_flag = Data(d['venue']['name'], d['venue']['state'] if is_aus else 'NZL',
+                                         raceNumber, arrowStartTimeLocal,
+                                         horseNumber, horseDisplayName, False)
+                print(data_without_flag)

requirements.txt

@@ -1,3 +1,4 @@
+arrow
 beautifulsoup4==4.7.1
 certifi==2019.6.16
 chardet==3.0.4