From 91517ccb811a878950e939303cc66b214038472c Mon Sep 17 00:00:00 2001 From: Sawood Alam Date: Tue, 17 May 2016 12:58:48 -0400 Subject: [PATCH 1/7] Correct the class name case --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ddce31f..487519c 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ 'si = timegate.examples.si:SiHandler', 'simple = timegate.examples.simple:ExampleHandler', 'w3c = timegate.examples.w3c:W3cHandler', - 'webcite = timegate.examples.webcite:WebciteHandler', + 'webcite = timegate.examples.webcite:WebCiteHandler', 'wikia = timegate.examples.wikia:WikiaHandler', 'wikipedia = timegate.examples.wikipedia:WikipediaHandler', ], From bdb6bacfc2f2f8514e8b11bc17a8c66eb510db28 Mon Sep 17 00:00:00 2001 From: Sawood Alam Date: Tue, 7 Jun 2016 15:21:01 -0400 Subject: [PATCH 2/7] Removing Portugese Web Archive Example --- setup.py | 1 - timegate/examples/po.py | 118 ---------------------------------------- 2 files changed, 119 deletions(-) delete mode 100644 timegate/examples/po.py diff --git a/setup.py b/setup.py index 487519c..0ee51d4 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,6 @@ 'nara = timegate.examples.nara:NaraHandler', 'orain = timegate.examples.orain:OrainHandler', 'pastpages = timegate.examples.pastpages:PastpagesHandler', - 'po = timegate.examples.po:PoHandler', 'sg = timegate.examples.sg:SgHandler', 'si = timegate.examples.si:SiHandler', 'simple = timegate.examples.simple:ExampleHandler', diff --git a/timegate/examples/po.py b/timegate/examples/po.py deleted file mode 100644 index 3f7fe7b..0000000 --- a/timegate/examples/po.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of TimeGate. -# Copyright (C) 2014, 2015 LANL. -# Copyright (C) 2016 CERN. -# -# TimeGate is free software; you can redistribute it and/or modify -# it under the terms of the Revised BSD License; see LICENSE file for -# more details. - -"""Memento proxy for Portuguese Web Archive arquivo.pt.""" - -from __future__ import absolute_import, print_function - -import StringIO -import logging -import re - -from lxml import etree - -from timegate.errors import HandlerError -from timegate.handler import Handler -from timegate.utils import get_uri_representations - - -def get_uri_representations(uri): - """Needed because some API do not process the url canonization properly, - and treat http://example.com and http://www.example.com differently. - - This function returns the list of canonical representation of a URI. - It also appends the `http://` protocol if missing. For instance, - *uri*='example.com' will return ['http://example.com', - 'http://www.example.com'] and *uri*='domain.example.com' will return - ['http://domain.example.com']. - - :param uri: The uri string to make canonical. - :return: The canonical representations of this URI. - """ - uri_list = [] - - m = re.match(r'(https?://)?(www.)?(.+)', uri) - if not m: - return uri_list - http, www, netloc = m.groups() - # domain.example.com/path/ not example.com/path/ - has_subdomain = len(netloc.split('/')[0].split('.')) > 2 - - if not http: - http = 'http://' - - if www: - uri_list.append(http + www + netloc) - if not has_subdomain: - uri_list.append(http + netloc) - else: - www = 'www.' - uri_list.append(http + netloc) - if not has_subdomain: - uri_list.append(http + www + netloc) - - return uri_list - - -class PoHandler(Handler): - - def __init__(self): - Handler.__init__(self) - self.baseuri = "http://arquivo.pt/wayback/wayback/xmlquery" - - def get_all_mementos(self, req_uri): - all_mementos = [] - [all_mementos.extend(self.query(uri)) - for uri in get_uri_representations(req_uri)] - return all_mementos - - def query(self, resource): - # implement the changes list for this particular proxy - param = {'url': resource, - 'type': 'urlquery'} - - changes = [] - - uri = self.baseuri + resource - dom = self.get_xml(uri, params=param) - if dom: - rlist = dom.xpath('/wayback/results/result') - for a in rlist: - dtstr = a.xpath('./capturedate/text()')[0] - url = a.xpath('./url/text()')[0] - loc = "http://arquivo.pt/wayback/wayback/%s/%s" % (dtstr, url) - - dtstr += " GMT" - changes.append((loc, dtstr)) - - return changes - - def get_xml(self, uri, params=None, html=False): - """Retrieves the resource using the url, parses it as XML or HTML and - returns the parsed dom object. - - :param uri: [str] The uri to retrieve :param headers: - [dict(header_name: value)] optional http headers to send in the - request :param html: [bool] optional flag to parse the response - as HTML :return: [lxml_obj] parsed dom. - - """ - - page = self.request(uri, params=params) - try: - page_data = page.content - if not html: - parser = etree.XMLParser(recover=True) - else: - parser = etree.HTMLParser(recover=True) - return etree.parse(StringIO.StringIO(page_data), parser) - except Exception: - logging.error("Cannot parse XML/HTML from %s" % uri) - raise HandlerError("Couldn't parse data from %s" % uri, 404) From 5be5167d100753c51e2fe7c2b94bc57acbe5121f Mon Sep 17 00:00:00 2001 From: Tibor Simko Date: Wed, 22 Jun 2016 15:55:05 +0200 Subject: [PATCH 3/7] docs: sphinx 1.4.2 Signed-off-by: Tibor Simko --- docs/conf.py | 12 +++--------- setup.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cb61f18..444beaa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,20 +14,14 @@ import sphinx.environment from docutils.utils import get_source_line - -def _warn_node(self, msg, node): - """Do not warn on external images.""" - if not msg.startswith('nonlocal image URI found:'): - self._warnfunc(msg, '{0}:{1}'.format(get_source_line(node))) - -sphinx.environment.BuildEnvironment.warn_node = _warn_node - - # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' +# Do not warn on external images. +suppress_warnings = ['image.nonlocal_uri'] + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. diff --git a/setup.py b/setup.py index 0ee51d4..492d709 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ 'ConfigParser>=3.3.0r2', ], 'docs': [ - 'Sphinx>=1.3', + 'Sphinx>=1.4.2', ], 'uwsgi': [ 'uWSGI>=2.0.3', From cf2702e05172d5387e72af0572b7cb3f0fd8e9f9 Mon Sep 17 00:00:00 2001 From: Harihar Shankar Date: Tue, 6 Sep 2016 13:09:27 -0600 Subject: [PATCH 4/7] inititalized the cache variable --- timegate/application.py | 1 + 1 file changed, 1 insertion(+) diff --git a/timegate/application.py b/timegate/application.py index 8cd2882..46ac181 100644 --- a/timegate/application.py +++ b/timegate/application.py @@ -106,6 +106,7 @@ def __init__(self, config=None, cache=None): self.config = Config(None) self.config.from_object(constants) self.config.update(config or {}) + self.cache = None if cache: self.cache = cache elif self.config['CACHE_USE']: From 92f0e0cbeebe5e514523ffa4a512460fdd8d0b03 Mon Sep 17 00:00:00 2001 From: Harihar Shankar Date: Mon, 3 Apr 2017 13:25:25 -0600 Subject: [PATCH 5/7] added support for user-agent header. fixes #25 --- timegate/application.py | 8 +++----- timegate/conf/config.ini | 8 ++++++-- timegate/conf/timegate.ini | 13 +++++++------ timegate/config.py | 17 ++++++++++++++++- timegate/examples/arxiv.py | 3 +-- timegate/handler.py | 8 +++++++- 6 files changed, 40 insertions(+), 17 deletions(-) diff --git a/timegate/application.py b/timegate/application.py index 46ac181..229c538 100644 --- a/timegate/application.py +++ b/timegate/application.py @@ -12,9 +12,6 @@ from __future__ import absolute_import, print_function -import glob -import importlib -import inspect import json import logging import os @@ -31,10 +28,8 @@ from werkzeug.wrappers import Request, Response from . import constants -from ._compat import quote, unquote from .cache import Cache from .config import Config -from .errors import TimegateError, URIRequestError from .handler import Handler, parsed_request from .utils import best @@ -47,6 +42,9 @@ request = local('request') """Proxy to request object.""" +#logging.getLogger(__name__) +#logging.basicConfig(level=logging.DEBUG) + def url_for(*args, **kwargs): """Proxy to URL Map adapter builder.""" diff --git a/timegate/conf/config.ini b/timegate/conf/config.ini index c71c4d9..e30dec7 100644 --- a/timegate/conf/config.ini +++ b/timegate/conf/config.ini @@ -14,11 +14,15 @@ strict_datetime = true # Timeout for any API request in seconds api_time_out = 6 +# user-agent +# Provide a user-agent to be added to the requests made by the timegate server +user_agent = Memento TimeGate + [handler] # handler_class # Optional path to handler class. If not provided the program will # search core extensions for a possible handler. -handler_class = timegate.examples.simple:ExampleHandler +handler_class = timegate.examples.es.EsHandler # use_timemap # Optional boolean to define wether the program can handle timemap requests. @@ -37,7 +41,7 @@ is_vcs = true # (Optional) String that will be prepended to requested URI if it is not already present # For example, if the server runs at `http://timegate.example.com` and all original resources begin with `http://example.com/res/{resource ID}`, # then setting `base_uri = http://example.com/res/` will allow short requests such `http://timegate.example.com/{resource ID}` -base_uri = http://www.example.com/ +base_uri = [cache] diff --git a/timegate/conf/timegate.ini b/timegate/conf/timegate.ini index 3cf9596..4e70be8 100644 --- a/timegate/conf/timegate.ini +++ b/timegate/conf/timegate.ini @@ -1,15 +1,16 @@ # uWSGI launch configuration file [uwsgi] -home = /data/venv/timegate/ -socket = /data/var/run/timegate/w3c/w3c.sock -chdir = /data/web/timegate/w3c +home = /Users/harihar/venv/timegate/ +#socket = uwsgi.sock +http = :9000 +#chdir = /data/web/timegate/w3c -daemonize = /data/var/logs/timegate/w3c.log +#daemonize = /data/var/logs/timegate/w3c.log module = timegate.application callable = application master = true -pidfile = /data/var/run/timegate/w3c/w3c.pid -harakiri = 120 +#pidfile = /data/var/run/timegate/w3c/w3c.pid +#harakiri = 120 memory-report processes = 4 diff --git a/timegate/config.py b/timegate/config.py index 8a051b3..484416a 100644 --- a/timegate/config.py +++ b/timegate/config.py @@ -19,8 +19,22 @@ class Config(dict): """Implement custom loaders to populate dict.""" + _instance = None + + def __new__(cls, root_path, defaults=None): + """ + Converting this into a singleton for cached access. + :param root_path: + :param defaults: + :return: + """ + if not cls._instance: + cls._instance = super(Config, cls).__new__(cls) + return cls._instance + def __init__(self, root_path, defaults=None): - """Build an empty config wrapper. + """ + Build an empty config wrapper. :param root_path: Path to which files are read relative from. :param defaults: An optional dictionary of default values. @@ -36,6 +50,7 @@ def from_inifile(self, filename, silent=True): # Server configuration self['HOST'] = conf.get('server', 'host').rstrip('/') + self['USER_AGENT'] = conf.get('server', 'user_agent') self['STRICT_TIME'] = conf.getboolean('server', 'strict_datetime') if conf.has_option('server', 'api_time_out'): self['API_TIME_OUT'] = conf.getfloat('server', 'api_time_out') diff --git a/timegate/examples/arxiv.py b/timegate/examples/arxiv.py index 670ab94..2bb2513 100644 --- a/timegate/examples/arxiv.py +++ b/timegate/examples/arxiv.py @@ -28,8 +28,7 @@ def __init__(self): Handler.__init__(self) # Resources - - # Ignores all that trails the identifier (? params, vX version,...) + # Ignores all that trails the identifier (? params, vX version,...info) self.rex = re.compile( r'(http://arxiv.org)/((?:pdf)|(?:abs))/(\d+\.\d+)(.*)') self.api_base = 'http://export.arxiv.org/oai2' diff --git a/timegate/handler.py b/timegate/handler.py index e3c8666..f695c0f 100644 --- a/timegate/handler.py +++ b/timegate/handler.py @@ -21,6 +21,7 @@ from ._compat import quote from .constants import API_TIME_OUT, TM_MAX_SIZE from .errors import HandlerError +from .config import Config class Handler(object): @@ -43,6 +44,11 @@ def request(self, resource, timeout=API_TIME_OUT, **kwargs): :raises HandlerError: if the requests fails to access the API. """ uri = resource + config = Config(None) + user_agent = config.get("USER_AGENT") + headers = {} + if user_agent: + headers["User-Agent"] = user_agent # Request logging with params try: @@ -55,7 +61,7 @@ def request(self, resource, timeout=API_TIME_OUT, **kwargs): logging.info('Sending request for %s' % uri) try: - req = requests.get(uri, timeout=timeout, **kwargs) + req = requests.get(uri, timeout=timeout, headers=headers, **kwargs) except Exception as e: logging.error('Cannot request server (%s): %s' % (uri, e)) raise HandlerError('Cannot request version server.', 502) From dfaf1056323aec0c7e2e36b0c6837ea26a3a578a Mon Sep 17 00:00:00 2001 From: Sawood Alam Date: Fri, 13 Jul 2018 09:01:34 -0400 Subject: [PATCH 6/7] Fix formatting issues to make tests pass --- timegate/application.py | 4 ++-- timegate/config.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/timegate/application.py b/timegate/application.py index 229c538..5f1bb57 100644 --- a/timegate/application.py +++ b/timegate/application.py @@ -42,8 +42,8 @@ request = local('request') """Proxy to request object.""" -#logging.getLogger(__name__) -#logging.basicConfig(level=logging.DEBUG) +# logging.getLogger(__name__) +# logging.basicConfig(level=logging.DEBUG) def url_for(*args, **kwargs): diff --git a/timegate/config.py b/timegate/config.py index 484416a..febda41 100644 --- a/timegate/config.py +++ b/timegate/config.py @@ -24,9 +24,9 @@ class Config(dict): def __new__(cls, root_path, defaults=None): """ Converting this into a singleton for cached access. - :param root_path: - :param defaults: - :return: + :param root_path: + :param defaults: + :return: """ if not cls._instance: cls._instance = super(Config, cls).__new__(cls) From 8c846a2739885865db98bf0e93241dfa6824cbcc Mon Sep 17 00:00:00 2001 From: Sawood Alam Date: Sat, 14 Jul 2018 01:20:59 -0400 Subject: [PATCH 7/7] Sort imports to fix isort errors --- timegate/application.py | 3 ++- timegate/handler.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/timegate/application.py b/timegate/application.py index 5f1bb57..aaa81d4 100644 --- a/timegate/application.py +++ b/timegate/application.py @@ -17,9 +17,10 @@ import os from datetime import datetime +from pkg_resources import iter_entry_points + from dateutil.tz import tzutc from link_header import Link, LinkHeader -from pkg_resources import iter_entry_points from werkzeug.exceptions import HTTPException, abort from werkzeug.http import http_date, parse_date from werkzeug.local import Local, LocalManager diff --git a/timegate/handler.py b/timegate/handler.py index f695c0f..f74a0bd 100644 --- a/timegate/handler.py +++ b/timegate/handler.py @@ -19,9 +19,9 @@ from . import utils as timegate_utils from ._compat import quote +from .config import Config from .constants import API_TIME_OUT, TM_MAX_SIZE from .errors import HandlerError -from .config import Config class Handler(object):