diff --git a/.gitignore b/.gitignore index 6d44d8b..ab9a95b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ COMMIT_EDITMSG #* .idea *.iml +gtfs.zip .DS_Store **/tests/*feed diff --git a/README.rst b/README.rst index 4106c59..3455054 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ -====== +=========== GTFSDB -====== +=========== + .. image:: https://badges.gitter.im/Join%20Chat.svg :alt: Join the chat at https://gitter.im/OpenTransitTools/gtfsdb @@ -8,86 +9,84 @@ GTFSDB Supported Databases -=================== +******************* -- PostgreSQL (PostGIS for Geo tables) - preferred -- Oracle - tested -- MySQL - tested -- SQLite - tested +* PostgreSQL (PostGIS for Geo tables) - preferred +* Oracle - tested +* MySQL - tested +* SQLite - tested GTFS (General Transit Feed Specification) Database -================================================== +************************************************** -Python code that will load GTFS data into a relational database, and SQLAlchemy ORM bindings to the GTFS tables in the gtfsdb. -The gtfsdb project's focus is on making GTFS data available in a programmatic context for software developers. The need for the -gtfsdb project comes from the fact that a lot of developers start out a GTFS-related effort by first building some amount of code -to read GTFS data (whether that's an in-memory loader, a database loader, etc...); GTFSDB can hopefully reduce the need for such -drudgery, and give developers a starting point beyond the first step of dealing with GTFS in .csv file format. +Python code that will load GTFS data into a relational database, and SQLAlchemy ORM bindings to the GTFS tables in the gtfsdb. The gtfsdb project's focus is on making GTFS data available in a programmatic context for software developers. The need for the gtfsdb project comes from the fact that a lot of developers start out a GTFS-related effort by first building some amount of code to read GTFS data (whether that's an in-memory loader, a database loader, etc...); GTFSDB can hopefully reduce the need for such drudgery, and give developers a starting point beyond the first step of dealing with GTFS in .csv file format. -(Slightly out-of-date version) available on pypi: https://pypi.python.org/pypi/gtfsdb +Available on pypi: https://pypi.python.org/pypi/gtfsdb Install from source via github (if you want the latest code) : -========================================== +************************************************************** + +#. Install Python 3.x https://www.python.org/downloads/ (code also runs on 2.7 if you are stuck on that version) +#. `pip install zc.buildout` - https://pypi.org/project/zc.buildout +#. (optinal step for **postgres users**: 'pip install psycopg2-binary') +#. git clone https://github.com/OpenTransitTools/gtfsdb.git +#. cd gtfsdb +#. buildout install prod -- NOTE: if you're using postgres, do a 'buildout install prod postgresql' +#. bin/gtfsdb-load --database_url +#. examples: -1. Install Python 3.x https://www.python.org/downloads/ (code also runs on 2.7 if you are stuck on that version) + * bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip -2. `pip install zc.buildout` - https://pypi.org/project/zc.buildout + * bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip -3. (optinal step for **postgres users**: 'pip install psycopg2-binary') + * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip -4. git clone https://github.com/OpenTransitTools/gtfsdb.git + .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... -5. cd gtfsdb +#. view db ( example: https://sqliteonline.com ) -6. buildout install prod -- NOTE: if you're using postgres, do a 'buildout install prod postgresql' +The best way to get gtfsbd up and running is via the 'zc.buildout' tool. Highly recommended to first install +buildout (e.g., pip install zc.buildout) before doing much of anything else. -7. bin/gtfsdb-load --database_url +Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning - examples: +.. note:: if you get the message "ImportError: No module named psycopg2", then 'pip install psychopg2-binary' should fix things. (Assumes you have postgres also installed on the machine you're trying to use the pg driver). - - bin/gtfsdb-load --database_url sqlite:///gtfs.db gtfsdb/tests/large-sample-feed.zip - - bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip - - bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip - NOTE: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqllite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc... +Usage with Docker: +****************** -8. view db ( example: https://sqliteonline.com ) +#. Build the image with `docker build -t gtfsdb .` +#. Run it with: -The best way to get gtfsbd up and running is via the 'zc.buildout' tool. Highly recommended to first install -buildout (e.g., pip install zc.buildout) before doing much of anything else. + .. code-block:: bash -Postgres users, gtfsdb requires the psycopg2-binary database driver. Installing that via `pip install psychopg2-binary` -will relieve gtfsdb from re-installing locally as part of the build. And if after the fact, you see *exceptions* mentioning -**"ImportError: No module named psycopg2"**, then 'pip install psychopg2-binary' should fix that up quick... + docker run gtfsdb --database_url + .. note:: The entrypoint command is `bin/gtfsdb-load` so the arguments will be passed to it. -Usage with Docker -================= -1. Build the image with :code:`docker build -t gtfsdb .`. -2. Run it with - :code:`docker run gtfsdb --database_url `. - The entrypoint command is :code:`bin/gtfsdb-load` - so the arguments will be passed to it. +Example Queries: +**************** +* get first stop time of each trip for route_id 1 -Example Query: -============== + .. code-block:: sql --- get first stop time of each trip for route_id 1 + select * + from trips t, stop_times st + where t.route_id = '1' + and t.trip_id = st.trip_id + and st.stop_sequence = 1 -select * -from trips t, stop_times st -where t.route_id = '1' -and t.trip_id = st.trip_id -and st.stop_sequence = 1 +* get agency name and number of routes --- get agency name and number of routes + .. code-block:: sql -select a.agency_name, a.agency_id, count(r.route_id) -from routes r, agency a -where r.agency_id = a.agency_id -group by a.agency_id, a.agency_name -order by 3 desc + select a.agency_name, a.agency_id, count(r.route_id) + from routes r, agency a + where r.agency_id = a.agency_id + group by a.agency_id, a.agency_name + order by 3 desc diff --git a/gtfsdb/__init__.py b/gtfsdb/__init__.py index 7a7ffcd..c6d1b94 100644 --- a/gtfsdb/__init__.py +++ b/gtfsdb/__init__.py @@ -15,6 +15,7 @@ from gtfsdb.model.stop_feature import * # noqa from gtfsdb.model.stop_time import StopTime # noqa from gtfsdb.model.transfer import Transfer # noqa +from gtfsdb.model.translation import Translation # noqa from gtfsdb.model.trip import Trip # noqa from gtfsdb.model.block import Block # noqa @@ -42,6 +43,7 @@ FareAttribute.__name__, FareRule.__name__, UniversalCalendar.__name__, + Translation.__name__, ] diff --git a/gtfsdb/model/stop.py b/gtfsdb/model/stop.py index 63374bc..8a9ff36 100644 --- a/gtfsdb/model/stop.py +++ b/gtfsdb/model/stop.py @@ -1,7 +1,7 @@ from collections import defaultdict from sqlalchemy import Column, Integer, Numeric, String -from sqlalchemy.orm import joinedload, joinedload_all, object_session, relationship +from sqlalchemy.orm import joinedload, object_session, relationship from gtfsdb import config, util from gtfsdb.model.base import Base @@ -82,7 +82,7 @@ def headsigns(self): session = object_session(self) log.info("QUERY StopTime") q = session.query(StopTime) - q = q.options(joinedload_all('trip.route')) + q = q.options(joinedload('trip').joinedload('route')) q = q.filter_by(stop_id=self.stop_id) for r in q: headsign = r.stop_headsign or r.trip.trip_headsign diff --git a/gtfsdb/model/stop_base.py b/gtfsdb/model/stop_base.py index 208b602..b0414e2 100644 --- a/gtfsdb/model/stop_base.py +++ b/gtfsdb/model/stop_base.py @@ -1,5 +1,5 @@ from sqlalchemy import Column, Integer, Numeric, String -from sqlalchemy.orm import joinedload, joinedload_all, object_session, relationship +from sqlalchemy.orm import joinedload, object_session from gtfsdb import config from gtfsdb.util import BBox, Point diff --git a/gtfsdb/model/stop_time.py b/gtfsdb/model/stop_time.py index 303ddbf..d7898bf 100644 --- a/gtfsdb/model/stop_time.py +++ b/gtfsdb/model/stop_time.py @@ -4,7 +4,7 @@ from gtfsdb import config, util from gtfsdb.model.base import Base from sqlalchemy import Column -from sqlalchemy.orm import joinedload_all, relationship +from sqlalchemy.orm import joinedload, relationship from sqlalchemy.sql.expression import func from sqlalchemy.types import SmallInteger, Integer, Numeric, String @@ -237,7 +237,7 @@ def get_departure_schedule(cls, session, stop_id, date=None, route_id=None, limi q = q.filter(StopTime.trip.has(Trip.route_id == route_id)) # step 3: options to speed up /q - q = q.options(joinedload_all('trip')) + q = q.options(joinedload('trip')) # step 4: order the stop times if limit is None or limit > 1: diff --git a/gtfsdb/model/translation.py b/gtfsdb/model/translation.py new file mode 100644 index 0000000..df238c6 --- /dev/null +++ b/gtfsdb/model/translation.py @@ -0,0 +1,21 @@ +from sqlalchemy import Column, Integer, Sequence +from sqlalchemy.types import String + +from gtfsdb import config +from gtfsdb.model.base import Base + + +class Translation(Base): + datasource = config.DATASOURCE_GTFS + filename = 'translations.txt' + + __tablename__ = 'translations' + + id = Column(Integer, Sequence(None, optional=True), primary_key=True) + table_name = Column(String(255), nullable=False) + field_name = Column(String(255), nullable=False) + language = Column(String(255), nullable=False) + translation = Column(String(255), nullable=False) + record_id = Column(String(255)) + record_sub_id = Column(String(255)) + field_value = Column(String(255)) diff --git a/setup.py b/setup.py index 1da39a1..396e0a4 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name='gtfsdb', - version='0.5.0', + version='0.6.0', description='GTFS Database', long_description=open('README.rst').read(), keywords='GTFS', @@ -40,10 +40,10 @@ ] }, classifiers=( - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', - 'License :: OSI Approved :: Mozilla-derived (MPL)', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.7',