marcus/PyJeeves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Started using 'sqlservice'. Added support for many new tables.

Browse source
This commit is contained in:
Marcus Lindvall 2019-07-04 14:25:42 +02:00
parent 9b7d7db996
commit b77a7069ce
12 changed files with 390 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -66,3 +66,5 @@ docs/_build
*.log
*.xls
config.yml

5
pyjeeves/__init__.py
View file

@ -1,6 +1 @@
# from logging.config import dictConfig
# import config
import logging

2
pyjeeves/config.py
View file

@ -64,7 +64,7 @@ try:
with open("config.yml", 'r') as ymlfile:
file_config = yaml.load(ymlfile, Loader=yaml.FullLoader)
config = {**config, **file_config} # Syntax introduced in Python 3.5
except IOError as e:
except IOError:
pass
dictConfig(config['logging'])

119
pyjeeves/models/abc.py
View file

@ -1,11 +1,12 @@
"""
Define an Abstract Base Class (ABC) for models
"""
from datetime import datetime
from decimal import Decimal
from sqlalchemy import inspect
from sqlalchemy.sql.expression import and_
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy.ext.hybrid import hybrid_property
from sqlservice import ModelBase
from pyjeeves import logging
@ -14,19 +15,25 @@ from . import db
logger = logging.getLogger("PyJeeves." + __name__)
class RawBaseModel():
class RawBaseModel(ModelBase):
""" Generalize __init__, __repr__ and to_json
Based on the models columns , ForetagKod=1"""
print_only = () # First filter
print_filter = () # Second filter
to_json_filter = () # Only json filter
column_map = {}
__to_dict_filter__ = []
__to_dict_only__ = ()
__column_map__ = {}
__table_args__ = {
'extend_existing': True
}
__dict_args__ = {
'adapters': {
# datetime: lambda value, col, *_: value.strftime('%Y-%m-%d'),
Decimal: lambda value, col, *_: "{:.2f}".format(value)
}
}
@classmethod
def _base_filters(self, obj, filters=and_()):
# This method provides base filtering, additional filtering can be done in subclasses
@ -38,63 +45,51 @@ class RawBaseModel():
filters
)
def __repr__(self):
""" Define a base way to print models
Columns inside `print_filter` are excluded """
return '%s(%s)' % (self.__class__.__name__, {
column: value
for column, value in self._to_dict().items()
if column not in self.print_filter
})
@staticmethod
def _to_json_types(value):
if isinstance(value, datetime):
return value.strftime('%Y-%m-%d')
if isinstance(value, Decimal):
return "%.2f" % value
try:
if isinstance(value, InstrumentedList):
return [x.json for x in value]
if type(value).__module__ != 'builtins': # Perhaps == builtin?
return value.json
except AttributeError:
logger.debug(str(type(value)) + " was not converted to jsonifyable type")
return None
return value
@property
def json(self):
""" Define a base way to jsonify models
Columns inside `to_json_filter` are excluded
Columns inside `to_json_only_filter` are only included """
return {
column: RawBaseModel._to_json_types(value)
# if not isinstance(value, datetime) else value.strftime('%Y-%m-%d')
# if type(value).__module__ != self.__module__ # Perhaps == builtin?
# else value.json # Convert instances to json if same module
for column, value in self._to_dict().items()
if column not in self.to_json_filter
}
def _to_dict(self):
""" This would more or less be the same as a `to_json`
But putting it in a "private" function
Allows to_json to be overriden without impacting __repr__
Or the other way around
And to add filter lists """
return {
self._map_columns(column.key): getattr(self, column.key)
for column in inspect(self.__class__).attrs
if not self.print_only or column.key in self.print_only
}
def _map_columns(self, key):
if key in self.column_map:
return self.column_map[key]
if key in self.__column_map__:
return self.__column_map__[key]
return key
def descriptors_to_dict(self):
"""Return a ``dict`` that maps data loaded in :attr:`__dict__` to this
model's descriptors. The data contained in :attr:`__dict__` represents
the model's state that has been loaded from the database. Accessing
values in :attr:`__dict__` will prevent SQLAlchemy from issuing
database queries for any ORM data that hasn't been loaded from the
database already.
Note:
The ``dict`` returned will contain model instances for any
relationship data that is loaded. To get a ``dict`` containing all
non-ORM objects, use :meth:`to_dict`.
Returns:
dict
"""
descriptors = self.descriptors()
return { # Expose hybrid_property extension
**{key: getattr(self, key) for key in descriptors.keys()
if isinstance(descriptors.get(key), hybrid_property)},
# and return all items included in descriptors
**{key: value for key, value in self.__dict__.items()
if key in descriptors}}
def to_dict(self):
rv = super().to_dict()
if self.__to_dict_only__:
return {
self._map_columns(key): rv[key]
for key in rv
if key in self.__to_dict_only__
}
for _filter in self.__to_dict_filter__:
rv.pop(_filter)
return rv
def merge(self):
db.raw_session.merge(self)
return self

239
pyjeeves/models/raw.py
View file

@ -10,9 +10,10 @@ from sqlalchemy.schema import MetaData, ForeignKey, Column
from sqlalchemy.orm import relationship
from sqlalchemy.types import Integer, String
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.sql.expression import and_
from sqlalchemy.exc import OperationalError
# from pyjeeves.session import raw_engine
from . import db
from pyjeeves import logging
@ -22,64 +23,153 @@ logger = logging.getLogger("PyJeeves." + __name__)
logger.info("Reading Jeeves DB structure")
meta = MetaData()
meta.reflect(bind=db.raw_session.connection(), only=['ar', 'ars', 'fr', 'kus', 'oh', 'lp', 'vg'])
# Table('fr', meta, implicit_returning=False)
try:
meta.reflect(bind=db.raw_session.connection(),
only=['ar', 'ars', 'xae', 'xare', 'fr', 'kus', 'x1k',
'oh', 'lp', 'vg', 'xp', 'xm', 'prh', 'prl'])
except OperationalError as e:
logger.error("Failed to read Jeeves DB structure")
raise e
Base = automap_base(cls=db.Model, name='Model', metadata=meta)
class ProductClass(Base, RawBaseModel):
__tablename__ = 'xp'
__column_map__ = {'ArtProdKlass': 'ProductClassNumber', 'ArtProdklBeskr': 'ProductClassName'}
__to_dict_only__ = ('ArtProdKlass', 'ArtProdklBeskr')
# print_filter = ('Articles', 'articles_collection')
class ArticleClass(Base, RawBaseModel):
__tablename__ = 'xm'
__column_map__ = {'ArtKod': 'ArticleClassNumber', 'ArtTypBeskr': 'ArticleClassName'}
__to_dict_only__ = ('ArtKod', 'ArtTypBeskr')
# print_filter = ('Articles', 'articles_collection')
class CommodityGroup(Base, RawBaseModel):
__tablename__ = 'vg'
column_map = {'VaruGruppKod': 'CommodityGroupNumber', 'VaruGruppBeskr': 'CommodityGroupName'}
print_only = ('VaruGruppKod', 'VaruGruppBeskr')
__column_map__ = {'VaruGruppKod': 'CommodityGroupNumber',
'VaruGruppBeskr': 'CommodityGroupName'}
__to_dict_only__ = ('VaruGruppKod', 'VaruGruppBeskr', 'ArticleClass')
print_filter = ('Articles', 'articles_collection')
# to_json_filter = ('Articles', 'articles_collection')
ArtKod = Column(Integer, ForeignKey('xm.ArtKod'), primary_key=True)
ArticleClass = relationship(ArticleClass)
class ArticleAlternativeUnit(Base, RawBaseModel):
__tablename__ = 'xae'
__column_map__ = {'AltEnhetKod': 'UnitCode', 'AltEnhetBeskr': 'UnitName',
'AltEnhetOmrFaktor': 'DefaultUnitConv'}
__to_dict_only__ = ('AltEnhetBeskr', 'AltEnhetOmrFaktor')
class ArticleUnit(Base, RawBaseModel):
__tablename__ = 'xare'
__column_map__ = {'ArtNr': 'ArticleNumber',
'AltEnhetKod': 'UnitCode', 'AltEnhetOmrFaktor': 'UnitConv',
'AltEnhetOrderStd': 'DefaultSalesUnit'}
__to_dict_only__ = ('AltEnhetKod', 'AltEnhetOmrFaktor',
'AltEnhetOrderStd', 'ArticleAlternativeUnit')
ArtNr = Column(String, ForeignKey('ar.ArtNr'), primary_key=True)
AltEnhetKod = Column(Integer, ForeignKey('xae.AltEnhetKod'), primary_key=True)
ArticleAlternativeUnit = relationship(ArticleAlternativeUnit)
class ArticleBalance(Base, RawBaseModel):
__tablename__ = 'ars'
column_map = {'LagSaldo': 'Balance',
'LagResAnt': 'ReservedBalance',
'LagsaldoAltEnh': 'BalanceAlternative',
'LagResAntAltEnh': 'ReservedAlternativeBalance',
'LagStalle': 'StorageLocationNumber'}
print_only = ('LagSaldo',
'LagResAnt',
'LagsaldoAltEnh',
'LagResAntAltEnh',
'LagStalle')
# print_filter = ('Articles', 'articles_collection')
# to_json_filter = ('Articles', 'articles_collection')
__column_map__ = {'LagSaldo': 'Balance',
'LagResAnt': 'ReservedBalance',
'LagsaldoAltEnh': 'BalanceAlternative',
'LagResAntAltEnh': 'ReservedAlternativeBalance',
'LagStalle': 'StorageLocationNumber'}
__to_dict_only__ = ('LagSaldo',
'LagResAnt',
'LagsaldoAltEnh',
'LagResAntAltEnh',
'LagStalle')
# print_filter = ('Article', 'articles_collection')
# to_json_filter = ('Article', 'articles_collection')
ArtNr = Column(Integer, ForeignKey('ar.ArtNr'), primary_key=True)
class Articles(Base, RawBaseModel):
class Article(Base, RawBaseModel):
__tablename__ = 'ar'
column_map = {'ArtNr': 'ArticleNumber',
'ArtBeskr': 'ArticleName',
'LagSaldoArtikel': 'Balance',
'EnhetsKod': 'Unit',
'ArtListPris': 'ListPrice'}
print_only = (
__column_map__ = {'ArtNr': 'ArticleNumber',
'ArtBeskr': 'ArticleName',
'ArtBeskrSpec': 'ArticleSpec',
'Edit': 'ArticleLongSpec',
'LagSaldoArtikel': 'UnitBalance',
'EnhetsKod': 'Unit',
'ArtListPris': 'UnitListPrice',
'Extra1': 'WholeSaleUnit'}
__to_dict_only__ = (
'ArtNr',
'ArtBeskr',
'ArtBeskrSpec',
'Edit',
'CommodityGroup',
'ProductClass',
'ArticleClass',
'ArticleBalance',
'EnhetsKod',
'LagSaldoArtikel',
'RowCreatedDt',
'ArtListPris')
'ArtListPris',
'PictureFileName',
'UnitListPrice',
'Extra1',
'ListPrice',
'Balance')
ArtNr = Column(Integer, primary_key=True)
VaruGruppKod = Column(Integer, ForeignKey('vg.VaruGruppKod'), primary_key=True)
ArtProdKlass = Column(Integer, ForeignKey('xp.ArtProdKlass'), primary_key=True)
ArtKod = Column(Integer, ForeignKey('xm.ArtKod'), primary_key=True)
CommodityGroup = relationship(CommodityGroup)
CommodityGroup = relationship(CommodityGroup, lazy='joined')
ProductClass = relationship(ProductClass, lazy='joined')
ArticleClass = relationship(ArticleClass, lazy='joined')
ArticleBalance = relationship(ArticleBalance)
ArticleUnit = relationship(ArticleUnit)
def get_unit_conv(self):
if self.ArtFsgForp:
return self.ArtFsgForp
for unit in self.ArticleUnit:
if unit.AltEnhetOrderStd == "1":
if unit.AltEnhetOmrFaktor:
return unit.AltEnhetOmrFaktor
else:
return unit.ArticleAlternativeUnit.AltEnhetOmrFaktor
return 1
@hybrid_property
def ListPrice(self):
try:
return self.ArtListPris * self.get_unit_conv()
except TypeError:
logger.debug("NoneType error, %s" % self.ArtNr)
@hybrid_property
def Balance(self):
try:
return self.LagSaldoArtikel / self.get_unit_conv()
except TypeError:
logger.debug("NoneType error, %s" % self.ArtNr)
@classmethod
def _base_filters(self, obj):
return RawBaseModel._base_filters(
@ -88,17 +178,102 @@ class Articles(Base, RawBaseModel):
)
class Companies(Base, RawBaseModel):
class Company(Base, RawBaseModel):
__tablename__ = 'fr'
column_map = {'FtgNr': 'CompanyNumber', 'FtgNamn': 'CompanyName'}
print_only = ('CompanyNumber', 'CompanyName')
__column_map__ = {'FtgNr': 'CompanyNumber', 'FtgNamn': 'CompanyName'}
__to_dict_only__ = ('FtgNr', 'FtgNamn', 'Customer')
FtgNr = Column(String, primary_key=True)
Customer = relationship('Customer', uselist=False, back_populates='Company', lazy='joined')
class CustomerCategory(Base, RawBaseModel):
__tablename__ = 'x1k'
KundKategoriKod = Column(Integer, primary_key=True)
class Customer(Base, RawBaseModel):
__tablename__ = 'kus'
__column_map__ = {'FtgNr': 'CompanyNumber', 'kundkategorikod': 'CustomerCategoryCode',
'PrisListaKundSpec': 'PriceListPrimary', 'PrisLista': 'PriceListSecondary'}
__to_dict_only__ = ('kundkategorikod', 'PriceList', 'PriceListCommon', 'CustomerCategory',
'PrisLista', 'PrisListaKundSpec')
FtgNr = Column(String, ForeignKey('fr.FtgNr'), primary_key=True)
KundKategoriKod = Column(Integer, ForeignKey('x1k.KundKategoriKod'))
PrisLista = Column(Integer, ForeignKey('prh.PrisLista'))
PrisListaKundSpec = Column(Integer, ForeignKey('prh.PrisLista'))
Company = relationship("Company", back_populates="Customer")
PriceList = relationship("PriceList", uselist=False,
lazy='joined', foreign_keys='PriceList.FtgNr')
PriceListCommon = relationship("PriceList", uselist=False,
foreign_keys='PriceList.PrisLista',
primaryjoin="Customer.PrisLista==PriceList.PrisLista")
KundKategori = relationship("CustomerCategory")
@hybrid_property
def CustomerCategory(self):
return self.KundKategori.KundKatBeskr
class PriceList(Base, RawBaseModel):
__tablename__ = 'prh'
__column_map__ = {'PrisListaBeskr': 'Description', 'PrisLista': 'PriceListNumber',
'MarkUpBelopp': 'PriceFactor'}
__to_dict_only__ = ('PrisListaBeskr', 'PrisLista', 'PriceListItems', 'MarkUpBelopp')
PrisLista = Column(Integer, primary_key=True)
FtgNr = Column(String, ForeignKey('kus.FtgNr'))
Customer = relationship('Customer', uselist=False, foreign_keys='Customer.PrisListaKundSpec')
PriceListItems = relationship('PriceListItem', back_populates="PriceList", lazy='joined')
class PriceListItem(Base, RawBaseModel):
__tablename__ = 'prl'
__column_map__ = {'ArtNr': 'ArticleNumber', 'vb_pris': 'UnitPrice',
'MarkUpBelopp': 'UnitPriceFactor', 'NollFaktor': 'NullPriceAllowed'}
__to_dict_only__ = ('ArtNr', 'vb_pris', 'MarkUpBelopp', 'NollFaktor', 'Price')
__to_dict_filter__ = ['PriceList']
# Do not serialize price list relationship
__dict_args__ = {
'adapters': {
**{
PriceList: None
},
**RawBaseModel.__dict_args__['adapters']
}
}
PrisLista = Column(Integer, ForeignKey('prh.PrisLista'), primary_key=True)
ArtNr = Column(Integer, ForeignKey('ar.ArtNr'), primary_key=True)
PriceList = relationship('PriceList', uselist=False)
Article = relationship(Article)
# TODO: Could likely be optimized by getting all articles in one query and mangled in repo
@hybrid_property
def Price(self):
if not self.vb_pris and not self.MarkUpBelopp:
return (
(self.Article.ArtListPris + self.PriceList.MarkUpBelopp) *
self.Article.get_unit_conv())
if self.vb_pris:
return self.vb_pris * self.Article.get_unit_conv()
else:
return (
(self.Article.ArtListPris + self.MarkUpBelopp) *
self.Article.get_unit_conv())
Base.prepare()
# Base companies for cusomters and suppliers
Customers = Base.classes.kus # Customer information
Orders = Base.classes.oh # Orders by customers
Order = Base.classes.oh # Orders by customers
DelivLoc = Base.classes.lp # Connections between a delivery company and customer company

4
pyjeeves/repositories/__init__.py
View file

@ -1,2 +1,4 @@
from .location import Location
from .article import Article
from .article import Article, ArticleCategory
from .company import Company
from .pricelist import PriceList

46
pyjeeves/repositories/article.py
View file

@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from pyjeeves.models.raw import Articles
from pyjeeves.models.raw import Article as ArticleModel, ProductClass, ArticleClass, CommodityGroup
from sqlalchemy.sql.expression import and_
from sqlalchemy.orm.exc import NoResultFound
from pyjeeves import logging
logger = logging.getLogger("PyJeeves." + __name__)
@ -9,36 +10,51 @@ logger = logging.getLogger("PyJeeves." + __name__)
# Relocate Jeeves modules to separate folder and let a "master" module handle imports, and setup.
class Article():
"""Handles dispatch locations in Jeeves"""
def __init__(self):
super(Article, self).__init__()
"""Handles articles in Jeeves"""
@staticmethod
def get(art_no):
""" Query an article by number """
return Articles.query.filter_by(
ArtNr=art_no
).one()
try:
return ArticleModel.query.filter_by(
ArtNr=art_no
).one()
except NoResultFound:
raise KeyError
@staticmethod
def get_all(filter_=and_(Articles.ItemStatusCode == 0, Articles.ArtKod != 2)):
def get_all(filter_=and_(ArticleModel.ItemStatusCode == 0, ArticleModel.ArtKod != 2)):
# .filter_by(ItemStatusCode=0, ArtKod=2)
return Articles.query.filter(filter_).all()
return ArticleModel.query.filter(filter_).all()
class ArticleCategory():
"""Handles article categories, such as classes and groups in Jeeves"""
@staticmethod
def get_all():
# .filter_by(ItemStatusCode=0, ArtKod=2)
prod_classes = ProductClass.query.all()
art_classes = ArticleClass.query.all()
com_groups = CommodityGroup.query.all()
return {'ProductClasses': prod_classes,
'ArticleClasses': art_classes, 'CommodityGroups': com_groups}
if __name__ == '__main__':
# print([column.key for column in Companies.__table__.columns])
# print([column.key for column in Company.__table__.columns])
logger.info("Starting TEST")
# session = RawSession()
logger.info("Testing gettings an article")
# c1 = session.query(Companies).filter_by(FtgNr="179580").first()
# print(Articles)
c1 = Articles.query.filter_by(ArtNr="2103").first()
# c1 = session.query(Company).filter_by(FtgNr="179580").first()
# print(ArticleModel)
c1 = ArticleModel.query.filter_by(ArtNr="2103").first()
print(c1)
logger.info(c1.json)
print (
len(Article.get_all())
print(
len(ArticleModel.get_all())
)

55
pyjeeves/repositories/company.py Normal file
View file

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
from pyjeeves.models.raw import Company as CompanyModel, Customer as CustomerModel
from sqlalchemy.sql.expression import and_
from pyjeeves import logging
logger = logging.getLogger("PyJeeves." + __name__)
# Relocate Jeeves modules to separate folder and let a "master" module handle imports, and setup.
class Company():
"""Handles companies in Jeeves"""
@staticmethod
def get(ftg_nr):
""" Query an article by number """
return CompanyModel.query.filter_by(
FtgNr=ftg_nr
).one()
@staticmethod
def get_all_active_customers():
cust = CustomerModel.query.filter(and_(CustomerModel.Makulerad == 0)).all()
return [c.CompanyModel for c in cust]
@staticmethod
def get_list(ftg_nr=[]):
return CompanyModel.query.filter(
CompanyModel.FtgNr.in_(ftg_nr)
).all()
if __name__ == '__main__':
# print([column.key for column in CompanyModel.__table__.columns])
logger.info("Starting TEST")
# session = RawSession()
logger.info("Testing gettings a company")
# c1 = session.query(CompanyModel).filter_by(FtgNr="179580").first()
# print(CompanyModel)
# print(CompanyModel.get_list(['406569', '179580', '2440070', '179584']))
from pprint import pprint
pprint(CompanyModel.get('179584').to_dict())
# c1 = CompanyModel.query.filter_by(FtgNr="406569").first()
# print(c1)
# logger.info(c1.json)
# print(
# len(CompanyModel.get_all_active_customers())
# )
# print(CompanyModel.get_all_active_customers()[0].CompanyModel)

17
pyjeeves/repositories/location.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
from pyjeeves.models.raw import Companies, DelivLoc
from pyjeeves.models.raw import Company, DelivLoc
from pyjeeves import logging
logger = logging.getLogger("PyJeeves." + __name__)
@ -9,6 +9,7 @@ logger = logging.getLogger("PyJeeves." + __name__)
# Relocate Jeeves modules to separate folder and let a "master" module handle imports, and setup.
class Location():
"""Handles dispatch locations in Jeeves"""
def __init__(self):
super(Location, self).__init__()
self.associated_company = '' # Company with new/existing locations
@ -30,7 +31,7 @@ class Location():
def create_lev_location(self, ftgnr='', name='', address='',
postal_code='', city='', gln='', invoice_ref='', phone=''):
_loc = Companies(
_loc = Company(
FtgNr=str(ftgnr), FtgNamn=name, FtgPostadr5=address,
FtgLevPostNr=postal_code, FtgPostLevAdr3=city,
EAN_Loc_Code=gln, FtgPostAdr1=invoice_ref, ComNr=phone,
@ -49,21 +50,21 @@ class Location():
for deliv_loc in self._deliv_locs:
deliv_loc.merge()
# self.session.merge(deliv_loc) # Create "connnections" between Customer and Location.
Companies.commit()
Company.commit()
# self.session.commit()
if __name__ == '__main__':
# print([column.key for column in Companies.__table__.columns])
# print([column.key for column in Company.__table__.columns])
logger.info("Starting TEST")
# session = RawSession()
logger.info("Testing gettings a company")
# c1 = session.query(Companies).filter_by(FtgNr="179580").first()
print(Companies)
c1 = Companies.query.filter_by(FtgNr="179580").first()
# c1 = session.query(Company).filter_by(FtgNr="179580").first()
print(Company)
c1 = Company.query.filter_by(FtgNr="179580").first()
logger.info(c1.json)
# RawSession.remove()
# from sqlalchemy.inspection import inspect
# print (inspect(Companies).columns.items())
# print (inspect(Company).columns.items())

22
pyjeeves/repositories/pricelist.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
from pyjeeves.models.raw import PriceList as PriceListModel
from sqlalchemy.orm.exc import NoResultFound
from pyjeeves import logging
logger = logging.getLogger("PyJeeves." + __name__)
# Relocate Jeeves modules to separate folder and let a "master" module handle imports, and setup.
class PriceList():
"""Handles price lists in Jeeves"""
@staticmethod
def get(price_list_no):
""" Query a price list by number """
try:
return PriceListModel.query.filter_by(
PrisLista=price_list_no
).one()
except NoResultFound:
raise KeyError

1
requirements.txt
View file

@ -2,6 +2,7 @@ nose==1.3.7
Sphinx==1.8.5
pymssql==2.1.4
SQLAlchemy==1.3.1
sqlservice==1.1.3
PyMySQL==0.9.3
alembic==1.0.8
PyYAML==5.1

1
setup.py
View file

@ -24,6 +24,7 @@ setup(
'sphinx',
'pymssql',
'sqlalchemy',
'sqlservice',
'PyMySQL',
'alembic',
'pyyaml',