From 5f550458b1a1b22aa5452b3e4f6b1626a3be52d6 Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 09:05:26 -0700 Subject: [PATCH 1/9] fix(sqlalchemy): get rid of depricated joinedload_all; bump versions, etc... --- .gitignore | 1 + gtfsdb/model/stop.py | 4 ++-- gtfsdb/model/stop_base.py | 2 +- gtfsdb/model/stop_time.py | 4 ++-- setup.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 6d44d8b..ab9a95b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ COMMIT_EDITMSG #* .idea *.iml +gtfs.zip .DS_Store **/tests/*feed diff --git a/gtfsdb/model/stop.py b/gtfsdb/model/stop.py index 63374bc..8a9ff36 100644 --- a/gtfsdb/model/stop.py +++ b/gtfsdb/model/stop.py @@ -1,7 +1,7 @@ from collections import defaultdict from sqlalchemy import Column, Integer, Numeric, String -from sqlalchemy.orm import joinedload, joinedload_all, object_session, relationship +from sqlalchemy.orm import joinedload, object_session, relationship from gtfsdb import config, util from gtfsdb.model.base import Base @@ -82,7 +82,7 @@ def headsigns(self): session = object_session(self) log.info("QUERY StopTime") q = session.query(StopTime) - q = q.options(joinedload_all('trip.route')) + q = q.options(joinedload('trip').joinedload('route')) q = q.filter_by(stop_id=self.stop_id) for r in q: headsign = r.stop_headsign or r.trip.trip_headsign diff --git a/gtfsdb/model/stop_base.py b/gtfsdb/model/stop_base.py index 208b602..b0414e2 100644 --- a/gtfsdb/model/stop_base.py +++ b/gtfsdb/model/stop_base.py @@ -1,5 +1,5 @@ from sqlalchemy import Column, Integer, Numeric, String -from sqlalchemy.orm import joinedload, joinedload_all, object_session, relationship +from sqlalchemy.orm import joinedload, object_session from gtfsdb import config from gtfsdb.util import BBox, Point diff --git a/gtfsdb/model/stop_time.py b/gtfsdb/model/stop_time.py index 303ddbf..d7898bf 100644 --- a/gtfsdb/model/stop_time.py +++ b/gtfsdb/model/stop_time.py @@ -4,7 +4,7 @@ from gtfsdb import config, util from gtfsdb.model.base import Base from sqlalchemy import Column -from sqlalchemy.orm import joinedload_all, relationship +from sqlalchemy.orm import joinedload, relationship from sqlalchemy.sql.expression import func from sqlalchemy.types import SmallInteger, Integer, Numeric, String @@ -237,7 +237,7 @@ def get_departure_schedule(cls, session, stop_id, date=None, route_id=None, limi q = q.filter(StopTime.trip.has(Trip.route_id == route_id)) # step 3: options to speed up /q - q = q.options(joinedload_all('trip')) + q = q.options(joinedload('trip')) # step 4: order the stop times if limit is None or limit > 1: diff --git a/setup.py b/setup.py index 1da39a1..625ffca 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name='gtfsdb', - version='0.5.0', + version='0.6.0', description='GTFS Database', long_description=open('README.rst').read(), keywords='GTFS', From 2610c397298572795cecca31911abc6cff9bfc55 Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 09:33:53 -0700 Subject: [PATCH 2/9] fix(setup.py): up lic and status --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 625ffca..396e0a4 100644 --- a/setup.py +++ b/setup.py @@ -40,10 +40,10 @@ ] }, classifiers=( - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', - 'License :: OSI Approved :: Mozilla-derived (MPL)', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.7', From b6713e006bd67f910bf391b2bfc9f0290fddab5c Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 09:51:49 -0700 Subject: [PATCH 3/9] fix(docs): fix up readme --- README.rst | 118 +++++++++++++++++++++++++---------------------------- 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/README.rst b/README.rst index 4106c59..cc931d5 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ -====== +=========== GTFSDB -====== +=========== + .. image:: https://badges.gitter.im/Join%20Chat.svg :alt: Join the chat at https://gitter.im/OpenTransitTools/gtfsdb @@ -8,52 +9,41 @@ GTFSDB Supported Databases -=================== +************ -- PostgreSQL (PostGIS for Geo tables) - preferred -- Oracle - tested -- MySQL - tested -- SQLite - tested +* PostgreSQL (PostGIS for Geo tables) - preferred +* Oracle - tested +* MySQL - tested +* SQLite - tested GTFS (General Transit Feed Specification) Database -================================================== +************ -Python code that will load GTFS data into a relational database, and SQLAlchemy ORM bindings to the GTFS tables in the gtfsdb. -The gtfsdb project's focus is on making GTFS data available in a programmatic context for software developers. The need for the -gtfsdb project comes from the fact that a lot of developers start out a GTFS-related effort by first building some amount of code -to read GTFS data (whether that's an in-memory loader, a database loader, etc...); GTFSDB can hopefully reduce the need for such -drudgery, and give developers a starting point beyond the first step of dealing with GTFS in .csv file format. +Python code that will load GTFS data into a relational database, and SQLAlchemy ORM bindings to the GTFS tables in the gtfsdb. The gtfsdb project's focus is on making GTFS data available in a programmatic context for software developers. The need for the gtfsdb project comes from the fact that a lot of developers start out a GTFS-related effort by first building some amount of code to read GTFS data (whether that's an in-memory loader, a database loader, etc...); GTFSDB can hopefully reduce the need for such drudgery, and give developers a starting point beyond the first step of dealing with GTFS in .csv file format. -(Slightly out-of-date version) available on pypi: https://pypi.python.org/pypi/gtfsdb +Available on pypi: https://pypi.python.org/pypi/gtfsdb Install from source via github (if you want the latest code) : -========================================== - -1. Install Python 3.x https://www.python.org/downloads/ (code also runs on 2.7 if you are stuck on that version) - -2. `pip install zc.buildout` - https://pypi.org/project/zc.buildout - -3. (optinal step for **postgres users**: 'pip install psycopg2-binary') - -4. git clone https://github.com/OpenTransitTools/gtfsdb.git - -5. cd gtfsdb - -6. buildout install prod -- NOTE: if you're using postgres, do a 'buildout install prod postgresql' - -7. bin/gtfsdb-load --database_url +************ - examples: +#. Install Python 3.x https://www.python.org/downloads/ (code also runs on 2.7 if you are stuck on that version) +#. `pip install zc.buildout` - https://pypi.org/project/zc.buildout +#. (optinal step for **postgres users**: 'pip install psycopg2-binary') +#. git clone https://github.com/OpenTransitTools/gtfsdb.git +#. cd gtfsdb +#. buildout install prod -- NOTE: if you're using postgres, do a 'buildout install prod postgresql' +#. bin/gtfsdb-load --database_url +#. examples: + * bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip + * bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip + * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip - - bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip - - bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip - - bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip +.. note:: + adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... - NOTE: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... - -8. view db ( example: https://sqliteonline.com ) +#. view db ( example: https://sqliteonline.com ) The best way to get gtfsbd up and running is via the 'zc.buildout' tool. Highly recommended to first install buildout (e.g., pip install zc.buildout) before doing much of anything else. @@ -64,30 +54,32 @@ will relieve gtfsdb from re-installing locally as part of the build. And if aft Usage with Docker -================= - -1. Build the image with :code:`docker build -t gtfsdb .`. -2. Run it with - :code:`docker run gtfsdb --database_url `. - The entrypoint command is :code:`bin/gtfsdb-load` - so the arguments will be passed to it. - - -Example Query: -============== - --- get first stop time of each trip for route_id 1 - -select * -from trips t, stop_times st -where t.route_id = '1' -and t.trip_id = st.trip_id -and st.stop_sequence = 1 - --- get agency name and number of routes - -select a.agency_name, a.agency_id, count(r.route_id) -from routes r, agency a -where r.agency_id = a.agency_id -group by a.agency_id, a.agency_name -order by 3 desc +************ + +#. Build the image with `docker build -t gtfsdb .` +#. Run it with: + .. code:: + docker run gtfsdb --database_url + + .. note:: + The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. + + +Example Queries: +************ + +* get first stop time of each trip for route_id 1 + .. code:: + select * + from trips t, stop_times st + where t.route_id = '1' + and t.trip_id = st.trip_id + and st.stop_sequence = 1 + +* get agency name and number of routes + .. code:: + select a.agency_name, a.agency_id, count(r.route_id) + from routes r, agency a + where r.agency_id = a.agency_id + group by a.agency_id, a.agency_name + order by 3 desc From 57a83e0af2e0beb92a96e4ad0acc2e2e4a807261 Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 09:55:38 -0700 Subject: [PATCH 4/9] fix(docs): fix up readme 2 --- README.rst | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index cc931d5..6e2325c 100644 --- a/README.rst +++ b/README.rst @@ -39,17 +39,14 @@ Install from source via github (if you want the latest code) : * bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip * bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip - -.. note:: - adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... + .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... #. view db ( example: https://sqliteonline.com ) The best way to get gtfsbd up and running is via the 'zc.buildout' tool. Highly recommended to first install buildout (e.g., pip install zc.buildout) before doing much of anything else. -Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` -will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning +Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning **"ImportError: No module named psycopg2"**, then 'pip install psychopg2-binary' should fix that up quick... @@ -58,18 +55,17 @@ Usage with Docker #. Build the image with `docker build -t gtfsdb .` #. Run it with: - .. code:: + .. code-block:: docker run gtfsdb --database_url - .. note:: - The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. + .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. Example Queries: ************ * get first stop time of each trip for route_id 1 - .. code:: + .. code-block:: select * from trips t, stop_times st where t.route_id = '1' @@ -77,7 +73,7 @@ Example Queries: and st.stop_sequence = 1 * get agency name and number of routes - .. code:: + .. code-block:: select a.agency_name, a.agency_id, count(r.route_id) from routes r, agency a where r.agency_id = a.agency_id From 4855014ea653fca667f996e39dfa83cead36098a Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 10:03:44 -0700 Subject: [PATCH 5/9] fix(docs): fix up readme 3 --- README.rst | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 6e2325c..fb1004e 100644 --- a/README.rst +++ b/README.rst @@ -36,10 +36,14 @@ Install from source via github (if you want the latest code) : #. buildout install prod -- NOTE: if you're using postgres, do a 'buildout install prod postgresql' #. bin/gtfsdb-load --database_url #. examples: + * bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip + * bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip + * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip - .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... + + .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... #. view db ( example: https://sqliteonline.com ) @@ -47,7 +51,8 @@ The best way to get gtfsbd up and running is via the 'zc.buildout' tool. Highly buildout (e.g., pip install zc.buildout) before doing much of anything else. Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning -**"ImportError: No module named psycopg2"**, then 'pip install psychopg2-binary' should fix that up quick... + +.. note:: **"ImportError: No module named psycopg2"**, then 'pip install psychopg2-binary' should fix that up quick... Usage with Docker @@ -55,7 +60,7 @@ Usage with Docker #. Build the image with `docker build -t gtfsdb .` #. Run it with: - .. code-block:: + .. code-block:: bash docker run gtfsdb --database_url .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. @@ -65,17 +70,20 @@ Example Queries: ************ * get first stop time of each trip for route_id 1 - .. code-block:: - select * - from trips t, stop_times st - where t.route_id = '1' - and t.trip_id = st.trip_id - and st.stop_sequence = 1 + .. code-block:: sql + + select * + from trips t, stop_times st + where t.route_id = '1' + and t.trip_id = st.trip_id + and st.stop_sequence = 1 * get agency name and number of routes - .. code-block:: - select a.agency_name, a.agency_id, count(r.route_id) - from routes r, agency a - where r.agency_id = a.agency_id - group by a.agency_id, a.agency_name - order by 3 desc + + .. code-block:: sql + + select a.agency_name, a.agency_id, count(r.route_id) + from routes r, agency a + where r.agency_id = a.agency_id + group by a.agency_id, a.agency_name + order by 3 desc From e1126ba2bb1a1eaecc1b9d0ab6d779b85d17254c Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 10:08:45 -0700 Subject: [PATCH 6/9] fix(docs): fix up readme 4 --- README.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index fb1004e..ada3722 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ Install from source via github (if you want the latest code) : * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip - .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... + .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... #. view db ( example: https://sqliteonline.com ) @@ -52,7 +52,7 @@ buildout (e.g., pip install zc.buildout) before doing much of anything else. Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning -.. note:: **"ImportError: No module named psycopg2"**, then 'pip install psychopg2-binary' should fix that up quick... +.. note:: if you get the message "ImportError: No module named psycopg2", then 'pip install psychopg2-binary' should fix things. (Assumes you have postgres also installed on the machine you're trying to use the pg driver). Usage with Docker @@ -60,8 +60,10 @@ Usage with Docker #. Build the image with `docker build -t gtfsdb .` #. Run it with: + .. code-block:: bash - docker run gtfsdb --database_url + + docker run gtfsdb --database_url .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. @@ -70,6 +72,7 @@ Example Queries: ************ * get first stop time of each trip for route_id 1 + .. code-block:: sql select * From f28aac7e13b0eff99cda5eb02932f7a6b428110d Mon Sep 17 00:00:00 2001 From: Frank Purcell Date: Mon, 17 Oct 2022 10:19:12 -0700 Subject: [PATCH 7/9] fix(docs): fix up readme 5 --- README.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index ada3722..3455054 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ GTFSDB Supported Databases -************ +******************* * PostgreSQL (PostGIS for Geo tables) - preferred * Oracle - tested @@ -18,7 +18,7 @@ Supported Databases GTFS (General Transit Feed Specification) Database -************ +************************************************** Python code that will load GTFS data into a relational database, and SQLAlchemy ORM bindings to the GTFS tables in the gtfsdb. The gtfsdb project's focus is on making GTFS data available in a programmatic context for software developers. The need for the gtfsdb project comes from the fact that a lot of developers start out a GTFS-related effort by first building some amount of code to read GTFS data (whether that's an in-memory loader, a database loader, etc...); GTFSDB can hopefully reduce the need for such drudgery, and give developers a starting point beyond the first step of dealing with GTFS in .csv file format. @@ -26,7 +26,7 @@ Available on pypi: https://pypi.python.org/pypi/gtfsdb Install from source via github (if you want the latest code) : -************ +************************************************************** #. Install Python 3.x https://www.python.org/downloads/ (code also runs on 2.7 if you are stuck on that version) #. `pip install zc.buildout` - https://pypi.org/project/zc.buildout @@ -55,21 +55,21 @@ Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing .. note:: if you get the message "ImportError: No module named psycopg2", then 'pip install psychopg2-binary' should fix things. (Assumes you have postgres also installed on the machine you're trying to use the pg driver). -Usage with Docker -************ +Usage with Docker: +****************** #. Build the image with `docker build -t gtfsdb .` #. Run it with: - .. code-block:: bash + .. code-block:: bash - docker run gtfsdb --database_url - - .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. + docker run gtfsdb --database_url + + .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. Example Queries: -************ +**************** * get first stop time of each trip for route_id 1 From 8daacc86380c8303e254c8f1223a7d06d51d669f Mon Sep 17 00:00:00 2001 From: Atsu06 Date: Fri, 31 Mar 2023 22:01:56 +0900 Subject: [PATCH 8/9] feat(translation):add translation.py to use translation table --- gtfsdb/model/translation.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 gtfsdb/model/translation.py diff --git a/gtfsdb/model/translation.py b/gtfsdb/model/translation.py new file mode 100644 index 0000000..df238c6 --- /dev/null +++ b/gtfsdb/model/translation.py @@ -0,0 +1,21 @@ +from sqlalchemy import Column, Integer, Sequence +from sqlalchemy.types import String + +from gtfsdb import config +from gtfsdb.model.base import Base + + +class Translation(Base): + datasource = config.DATASOURCE_GTFS + filename = 'translations.txt' + + __tablename__ = 'translations' + + id = Column(Integer, Sequence(None, optional=True), primary_key=True) + table_name = Column(String(255), nullable=False) + field_name = Column(String(255), nullable=False) + language = Column(String(255), nullable=False) + translation = Column(String(255), nullable=False) + record_id = Column(String(255)) + record_sub_id = Column(String(255)) + field_value = Column(String(255)) From 632c4305b63fc2d9f4ef6c414bb30bf24c85e125 Mon Sep 17 00:00:00 2001 From: Atsu06 Date: Fri, 31 Mar 2023 22:03:23 +0900 Subject: [PATCH 9/9] feat(translation):add to import and SORTED_CLASS_NAMES to use translation table --- gtfsdb/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gtfsdb/__init__.py b/gtfsdb/__init__.py index 7a7ffcd..c6d1b94 100644 --- a/gtfsdb/__init__.py +++ b/gtfsdb/__init__.py @@ -15,6 +15,7 @@ from gtfsdb.model.stop_feature import * # noqa from gtfsdb.model.stop_time import StopTime # noqa from gtfsdb.model.transfer import Transfer # noqa +from gtfsdb.model.translation import Translation # noqa from gtfsdb.model.trip import Trip # noqa from gtfsdb.model.block import Block # noqa @@ -42,6 +43,7 @@ FareAttribute.__name__, FareRule.__name__, UniversalCalendar.__name__, + Translation.__name__, ]