Merge pull request #539 from ispras/heavy_refactor

Version 2.0

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+.cache
 .idea/
 .virtual_env/
 virtual_env/
@@ -19,3 +20,8 @@ alembic.ini
 
 # webui2
 target/
+
+Session.vim
+pytest.output
+sphinx/source/apidoc-*/
+sphinx/pyramid_autodoc

README.md

@@ -88,7 +88,7 @@ If you are using less recent distro version, replace `xenial` with `trusty` or `
              * `effective_cache_size` - 1/8 of total RAM (e.g. 4096MB)
              * `wal_level = replica`
      * Restart postgres: `sudo service postgresql restart`
-     * Import the latest database backup (from-scratch creation look in `# from psql` section): `sudo -u posgres psql < lingvodoc.sql`
+     * Import the latest database backup (from-scratch creation look in `# from psql` section): `sudo -u postgres psql lingvodoc < lingvodoc.sql`
 2. Install Redis: `sudo apt-get install redis-server`.
 3. Install compilers, libraries and git: `sudo apt-get install build-essential python3-dev libssl-dev git`
 4. Install venv-creator: `sudo apt-get install python3-virtualenv`
@@ -113,7 +113,7 @@ If you use this step, login as that user for all the other steps.
      proc_name = lingvodoc
      bind = unix:/tmp/lingvodoc.sock
      ``` 
-     (note: you should install gunicorn to use that config; it's not present in requirements since pserve/waitress works well too)
+     (note: you should install gunicorn to use that config; it`s not present in requirements since pserve/waitress works well too)
      (note2: if you are using that way, you will need wsgi-frontend [nginx for example])
     * section [app:main] 
       - `secret = 'your random string'`
@@ -139,8 +139,9 @@ If you use this step, login as that user for all the other steps.
       auth_version = 2.0
       tenant_name = admin
       ```
-14. Run lingvodoc: `pserve --daemon ./postgres.ini start`   
-             
+14. Run lingvodoc: `pserve --daemon ./postgres.ini start`
+15. (optionally) To run a celery worker you need to run `celery worker -A lingvodoc.queue.celery` from lingvodoc root.
+    * Enable the celery — open your celery.ini file and set the value of celery to `true`.
              
 Installing as desktop (user) environment for Ubuntu
 ---------------
@@ -163,3 +164,34 @@ API documentation
 /authors
 /sound
 /markup
+
+Testing
+-------
+
+Testing is performed via [pytest](http://doc.pytest.org). Installation of required packages:
+```
+pip install -r develop-requirements.txt
+```
+
+Running tests from the root project directory:
+```
+$VENV/bin/py.test
+```
+
+Documentation
+-------------
+
+Documentation for Python source code, including tests, can be generated via [Sphinx](http://www.sphinx-doc.org). Installation of required packages:
+```
+pip install -r develop-requirements.txt
+```
+
+Generating documentation from the root project directory:
+```
+make -f sphinx/Makefile html
+```
+
+Generated documentation can be accessed at `sphinx/build/html/index.html`. It includes all Python packages, modules, classes and functions of the project; descriptions of packages, modules, classes and functions are automatically produced from their docstrings.
+
+Documentation also includes an up-to-date list of Pyramid REST API routes provided by the application, extracted via [pyramid\_autodoc](https://github.com/SurveyMonkey/pyramid_autodoc) Sphinx extension.
+

alembictests.ini

@@ -7,7 +7,7 @@
 # path to migration scripts
 script_location = alembic
 
-sqlalchemy.url =  postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
+sqlalchemy.url =  postgresql+psycopg2://postgres@/lingvodoc_testing
 ;sqlalchemy.url = sqlite:////home/student/projects/lingvodoc/lingvodoc_testing.sqlite
 ;sqlalchemy.url = sqlite:///lingvodoc_testing.sqlite
 
@@ -50,7 +50,7 @@ pyramid.includes =
     pyramid_debugtoolbar
     pyramid_tm
 
-sqlalchemy.url = postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
+sqlalchemy.url = postgresql+psycopg2://postgres@/lingvodoc_testing
 ;sqlalchemy.url = sqlite:////home/student/projects/lingvodoc/lingvodoc_testing.sqlite
 ;sqlalchemy.url = sqlite:///lingvodoc_testing.sqlite
 
@@ -63,15 +63,15 @@ sqlalchemy.url = postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
 ###
 
 [server:main]
-;use = egg:waitress#main
-use = egg:gunicorn#main
-workers = 3
-timeout = 3000
-proc_name = lingvodoc
-bind = "0.0.0.0:6543"
+use = egg:waitress#main
+;use = egg:gunicorn#main
+;workers = 3
+;timeout = 3000
+;proc_name = lingvodoc
+;bind = "0.0.0.0:6543"
 host = 0.0.0.0
 port = 6543
-max_request_body_size = 2147483648
+;max_request_body_size = 2147483648
 
 ###
 # logging configuration
@@ -154,17 +154,17 @@ no-orphans = true
 
 log-slow = true
 
-virtualenv = /Users/al/environments/lingvodocenv/
+#virtualenv = /Users/al/environments/lingvodocenv/
 
 callable = lingvodoc
 
 
-[cache:dogpile]
-backend = dogpile.cache.redis
+#[cache:dogpile]
+#backend = dogpile.cache.redis
 
-[cache:dogpile:args]
-host: localhost
-port: 6379
-db: 0
-redis_expiration_time: 60
-distributed_lock: True
+#[cache:dogpile:args]
+#host: localhost
+#port: 6379
+#db: 0
+#redis_expiration_time: 60
+#distributed_lock: True

develop-requirements.txt

@@ -0,0 +1,4 @@
+Sphinx
+pytest
+simplejson
+webtest

development.ini

@@ -6,6 +6,8 @@
 [app:main]
 use = egg:lingvodoc
 
+secret = 'secret string'
+
 pyramid.reload_templates = true
 pyramid.debug_authorization = false
 pyramid.debug_notfound = false
@@ -15,7 +17,7 @@ pyramid.includes =
     pyramid_debugtoolbar
     pyramid_tm
 
-sqlalchemy.url = sqlite:///%(here)s/lingvodoc.sqlite
+sqlalchemy.url = postgresql+psycopg2://postgres@/lingvodoc
 
 # By default, the toolbar only appears for clients from IP addresses
 # '127.0.0.1' and '::1'.
@@ -39,14 +41,14 @@ port = 6543
 keys = root, lingvodoc, sqlalchemy
 
 [handlers]
-keys = console
+keys = console, filelog
 
 [formatters]
 keys = generic
 
 [logger_root]
-level = INFO
-handlers = console
+level = DEBUG
+handlers = console, filelog
 
 [logger_lingvodoc]
 level = DEBUG
@@ -67,6 +69,12 @@ args = (sys.stderr,)
 level = NOTSET
 formatter = generic
 
+[handler_filelog]
+class = handlers.RotatingFileHandler
+args = ('%(here)s/pyramid.log', 'a', 16777216, 1)
+level = NOTSET
+formatter = generic
+
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
@@ -78,21 +86,50 @@ administrator_password = password
 
 [backend:storage]
 # disk or openstack
-storagetype = disk
-path = /tmp
-authurl = http://10.10.10.121:5000/v2.0
-store = http://adelaide.intra.ispras.ru/horizon/project/containers
-user = admin
-key = tester
-auth_version = 2.0
-tenant_name = admin
+type = disk
+path = /tmp/
+prefix = http://localhost:6543/
+static_route = objects/
+#authurl = http://10.10.10.121:5000/v2.0
+#store = http://adelaide.intra.ispras.ru/horizon/project/containers
+#user = admin
+#key = tester
+#auth_version = 2.0
+#tenant_name = admin
+
+[uwsgi]
+socket = 0.0.0.0:6543
+protocol = http
+master = true
+
+processes = 4
+
+harakiri = 60
+harakiri-verbose = true
+limit-post = 65536
+post-buffering = 8192
+
+listen = 256
+
+max-requests = 1000
+
+reload-on-as = 128
+reload-on-rss = 96
+no-orphans = true
+
+log-slow = true
+
+callable = lingvodoc
 
+# Caching.
 [cache:dogpile]
+expiration_time = 36000
 backend = dogpile.cache.redis
+;backend = dogpile.cache.memcached
 
 [cache:dogpile:args]
-host: localhost
-port: 6379
-db: 0
-redis_expiration_time: 60
-distributed_lock: True
+;redis_expiration_time = 60*60*2
+host = localhost
+port = 6379
+db = 0
+distributed_lock = True

lingvodoc/__init__.py

@@ -24,6 +24,7 @@ def configure_routes(config):
     """
     This function registers views from .views for pyramid event loop. Actually all listed items here is our
     'site-map' for:
+
     1) web-views (html+javascript) - will be marked as 'web-view'
     2) REST API - will be marked as 'API'
     3) internal calls for frontend (in most cases it's API but not always). If it's not API part, will be marked
@@ -331,7 +332,7 @@ def configure_routes(config):
     # API #GET
     # no params, returns file
     config.add_route(name="get_user_blob",
-                     pattern="/blobs/{client_id}/{object_id}")
+                     pattern="/blobs/{client_id}/{object_id}", request_method='GET')
     # API #GET
     # no params, lists only own blobs
     config.add_route(name="list_user_blobs",
@@ -357,7 +358,7 @@ def configure_routes(config):
     # API #GET && DELETE
     # {entity_type: <entity_type>, content: <tag>, connections: [{object_id: <obj_id>, client_id: <cl_id>}
     config.add_route(name='get_group_entity', pattern='/group_entity/{client_id}/{object_id}',
-                     factory='lingvodoc.models.PerspectiveEntityAcl')  # tested (no del)
+                     factory='lingvodoc.models.LexicalViewAcl')  # tested (no del)
     # tags are different there and in connected words
 
     # API #GET
@@ -655,6 +656,17 @@ def configure_routes(config):
 
     config.add_route(name='translation_service_search', pattern='/translation_service_search')
 
+    config.add_route(name='sociolinguistics', pattern='/sociolinguistics')
+
+    config.add_route(name='sociolinguistics_questions', pattern='/sociolinguistics/questions')
+
+    config.add_route(name='sociolinguistics_answers', pattern='/sociolinguistics/answers')
+
+    config.add_route(name="delete_user_blob",
+                     pattern="/blobs/{client_id}/{object_id}", request_method='DELETE')
+
+    config.add_route(name="phonology", pattern="/phonology")
+
 
 def main(global_config, **settings):
     """ This function returns a Pyramid WSGI application.
@@ -667,15 +679,19 @@ def main(global_config, **settings):
     config_file = global_config['__file__']
     parser = ConfigParser()
     parser.read(config_file)
+
     # TODO: DANGER
-    storage = dict()
-    for k, v in parser.items('backend:storage'):
-        storage[k] = v
-    settings['storage'] = storage
+
+    settings['storage'] = dict(parser.items(
+        'backend:storage' if parser.has_section('backend:storage') else
+            'storage'))
+
     if parser.has_section('app:desktop'):
+        storage = dict()
         for k, v in parser.items('app:desktop'):
             storage[k] = v
         settings['desktop'] = storage
+
     config = Configurator(settings=settings)
     log = logging.getLogger(__name__)
 

lingvodoc/acl.py

@@ -13,6 +13,7 @@
 from sqlalchemy.orm import joinedload
 
 import logging
+from time import time
 log = logging.getLogger(__name__)
 
 
@@ -33,15 +34,14 @@ def groupfinder(client_id, request):
         return []
     try:
         user = DBSession.query(User) \
-                        .options(joinedload('groups').joinedload('BaseGroup')) \
-                        .options(joinedload('organizations').joinedload('groups').joinedload('BaseGroup')) \
                         .join(Client) \
                         .filter(Client.id == client_id).first()
 
         groups = DBSession.query(Group)\
+            .join(BaseGroup)\
+            .join(Group.users)\
             .options(joinedload('BaseGroup')) \
-            .join(BaseGroup) \
-            .filter(Group.users.contains(user))
+            .filter(User.id == user.id)
         if subject:
             groups = groups.filter(BaseGroup.subject == subject)
         groups = groups.all()

lingvodoc/cache/caching.py

@@ -5,7 +5,9 @@
 from lingvodoc.cache.mock.cache import MockCache
 
 
-MEMOIZE = None
+# We initialize MEMOIZE to identity function so that if the cache is not initialized (e.g. when an
+# automatically extracted source code documentation is being compiled), it is still possible to use it.
+MEMOIZE = lambda x: x
 CACHE = None
 
 
@@ -52,3 +54,4 @@ def initialize_cache(args):
     region = make_region().configure(**args)
     MEMOIZE = cache_responses(region)
     CACHE = CommonCache(region)
+