Unfloyded/Heliotrope
Archived
0
0
Fork 0
Code Activity

feat: apply hitomi changed (#171)

Browse source 
* fix(interpreter): remove statement (#168)

* refactor(table): hitomi changes applied

* refactor(domain): hitomi changes applied

* refactor(table): hitomi changes applied

* refactor(types): hitomi changes applied

* chore(vscode): add snippets

* refactor(orm): hitomi changes applied

* feat(odm): migration to mongodb (#169)

* feat(odm): migration to mongodb

* feat(config): add config for atlas search

* feat: apply changed

* chore(deps): add dependency

* test: apply changed

* test(config): apply changed

* chore(deps): bump up version motor

* feat!(parser): remove parser

* feat(interpreter): add get thumbnail method

* feat!(info): using galleryinfo

* feat(functions): add getthumbnail method

* feat!(hitomi): remove get info method

* fix(image): now return only webp or avif

* refactor(info): apply hitomi changed

* refactor(mirroring): apply hitomi changed

* fix(types): thumbnail is file

* test(common): edit dict

* style: apply code style

* feat(info): add from dict method

* style(info): fix type issue

* test(arg): fix test

* style(info): apply isort

* fix(domain): init false

* test(common): edit info

* test(conftest): edit image url

* test(mirroring): fix test

* fix(function): add base

* test(conftest): edit image url fixture

* test(conftest): fix conftest

* test(conftest): fix conftest
This commit is contained in:
Ryu juheon 2022-02-03 19:41:56 +09:00 • committed by GitHub
parent 97625b0223
commit a03b0f3c19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 1275 additions and 544 deletions
Download patch file Download diff file Expand all files Collapse all files

20
.github/workflows/ci.yml vendored
View file

@ -56,10 +56,18 @@ jobs:
POSTGRES_DB: test_heliotrope
ports:
- 127.0.0.1:5432:5432
meilisearch:
image: getmeili/meilisearch
mongo:
image: mongo
env:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: test
ports:
- 7700:7700
- 27017:27017
options:
--health-cmd mongo
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
@ -71,6 +79,12 @@ jobs:
python -m pip install -U pip
pip install -r requirements/deps.txt
pip install -r requirements/test.txt
- name: Make MongoDB collection
run: |
mongo -u root -p test --eval "db.getSiblingDB('hitomi').createCollection('info')"
- name: Make collection index
run: |
mongo -u root -p test --eval "db.getSiblingDB('hitomi').getCollection('info').createIndex({'title':'text'}, {'language_override': 'korean'})"
- name: Run tests
run: |
pytest --cov=heliotrope --cov-report=xml

10
.vscode/foreignkey.code-snippets vendored Normal file
View file

@ -0,0 +1,10 @@
{
"Foreignkey column": {
"scope": "python",
"prefix": "foreignkey column",
"body": [
" Column(\"galleryinfo_id\", Integer, ForeignKey(\"galleryinfo.id\")),",
],
"description": "Make new endpoint"
}
}

32
.vscode/license.code-snippets vendored Normal file
View file

@ -0,0 +1,32 @@
{
"Add LICENSE header": {
"scope": "",
"prefix": "add license header",
"body": [
"\"\"\"",
"MIT License",
"",
"Copyright (c) 2021 SaidBySolo",
"",
"Permission is hereby granted, free of charge, to any person obtaining a copy",
"of this software and associated documentation files (the \"Software\"), to deal",
"in the Software without restriction, including without limitation the rights",
"to use, copy, modify, merge, publish, distribute, sublicense, and/or sell",
"copies of the Software, and to permit persons to whom the Software is",
"furnished to do so, subject to the following conditions:",
"",
"The above copyright notice and this permission notice shall be included in all",
"copies or substantial portions of the Software.",
"",
"THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR",
"IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,",
"FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE",
"AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER",
"LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,",
"OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE",
"SOFTWARE.",
"\"\"\""
],
"description": "Log output to console"
}
}

19
.vscode/make_table.code-snippets vendored Normal file
View file

@ -0,0 +1,19 @@
{
"Make table": {
"scope": "python",
"prefix": "make table",
"body": [
"from sqlalchemy.sql.schema import Column, Table",
"",
"from heliotrope.database.orm.base import mapper_registry",
"from sqlalchemy.sql.sqltypes import Integer",
"",
"${1:${TM_FILENAME_BASE}}_table = Table(",
" \"${1:${TM_FILENAME_BASE}}\",",
" mapper_registry.metadata,",
" Column(\"id\", Integer, primary_key=True, autoincrement=True),",
")"
],
"description": "Make new endpoint"
}
}

10
.vscode/url_column.code-snippets vendored Normal file
View file

@ -0,0 +1,10 @@
{
"Url column": {
"scope": "python",
"prefix": "url column",
"body": [
" Column(\"url\", String, nullable=False),",
],
"description": "Make new endpoint"
}
}

6
heliotrope/abc/database.py
View file

@ -39,9 +39,9 @@ class AbstractInfoDatabase(ABC):
"""
@abstractmethod
async def add_infos(self, infos: list[Info]) -> None:
async def add_info(self, info: Info) -> None:
"""
Add infos to the database.
Add info to the database.
"""
@abstractmethod
@ -65,7 +65,7 @@ class AbstractInfoDatabase(ABC):
@abstractmethod
async def search(
self,
tags: list[str],
querys: list[str],
offset: int = 0,
limit: int = 15,
) -> tuple[list[Info], int]:

6
heliotrope/argparser.py
View file

@ -125,6 +125,12 @@ def parse_args(argv: list[str]) -> Namespace:
default="",
help="The secret to use for forwarded headers (default: '')",
)
config.add_argument(
"--use-atlas-search",
action="store_true",
default=False,
help="Use mongodb Atlas search (default: False)",
)
config.add_argument(
"--config",

4
heliotrope/config.py
View file

@ -49,11 +49,11 @@ class HeliotropeConfig(Config):
"SENTRY_DSN": "",
"GALLERYINFO_DB_URL": "",
"INFO_DB_URL": "",
"INFO_DB_API_KEY": "",
"INDEX_FILE": "index-korean.nozomi",
"MIRRORING_DELAY": 3600,
"REFRESH_COMMON_JS_DELAY": 86400,
"SUPERVISOR_DELAY": 30,
"USE_ATLAS_SEARCH": False,
# Sanic config
"HOST": "127.0.0.1",
"PORT": 8000,
@ -92,11 +92,11 @@ class HeliotropeConfig(Config):
SENTRY_DSN: str
GALLERYINFO_DB_URL: str
INFO_DB_URL: str
INFO_DB_API_KEY: str
MIRRORING_DELAY: float
REFRESH_COMMON_JS_DELAY: float
SUPERVISOR_DELAY: float
INDEX_FILE: str
USE_ATLAS_SEARCH: bool
# Sanic config
DEBUG: bool
HOST: str

140
heliotrope/database/meilisearch/__init__.py
View file

@ -1,140 +0,0 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from random import randrange
from typing import Optional, cast
from ameilisearch.client import Client
from ameilisearch.errors import MeiliSearchApiError
from ameilisearch.index import Index
from sanic.log import logger
from heliotrope.abc.database import AbstractInfoDatabase
from heliotrope.domain.info import Info
from heliotrope.types import HitomiInfoJSON
from heliotrope.utils import is_the_first_process
class MeiliSearch(AbstractInfoDatabase):
def __init__(self, client: Client, index: Index) -> None:
self.client = client
self.index = index
async def close(self) -> None:
logger.debug(f"close {self.__class__.__name__}")
await self.client.close()
await self.index.close()
@classmethod
async def setup(
cls, url: str, api_key: Optional[str] = None, uid: str = "hitomi"
) -> "MeiliSearch":
logger.debug(f"Setting up {cls.__name__}")
async with Client(url, api_key) as client:
task = await client.create_index(uid)
await client.wait_for_task(task["uid"])
async with await client.get_index(uid) as index:
if is_the_first_process:
await index.update_filterable_attributes(
[
"tags",
"artist",
"group",
"type",
"language",
"series",
"character",
]
)
await index.update_sortable_attributes(["id"])
await index.update_searchable_attributes(["title"])
instance = cls(client, index)
return instance
def parse_query(self, querys: list[str]) -> tuple[str, list[str]]:
# Tags are received in the following format: female:big_breasts
# If it is not in the following format, it is regarded as a title.
# 태그는 다음과 같은 형식으로 받아요: female:big_breasts
# 만약 다음과 같은 형식이 아니라면 제목으로 간주해요.
parsed_query: list[str] = []
title = ""
for query in querys:
if any(info_tag in query for info_tag in self.info_tags):
splited = query.split(":")
parsed_query.append(f"{splited[0]} = '{splited[1]}'")
elif any(
gender_common_tag in query
for gender_common_tag in self.gender_common_tags
):
parsed_query.append(f"tags = '{query}'")
else:
title = query
return title, parsed_query
async def get_total(self) -> int:
stats = await self.index.get_stats()
return int(stats["numberOfDocuments"])
async def get_all_index(self) -> list[int]:
total = await self.get_total()
results = await self.index.get_documents({"limit": total})
return list(map(lambda d: int(d["id"]), results))
async def add_infos(self, infos: list[Info]) -> None:
await self.index.add_documents([dict(info.to_dict()) for info in infos])
async def get_info(self, id: int) -> Optional[Info]:
try:
d = cast(HitomiInfoJSON, await self.index.get_document(str(id)))
except MeiliSearchApiError:
return None
return Info.from_dict(d)
async def get_info_list(self, offset: int = 0, limit: int = 15) -> list[Info]:
response = await self.index.search(
"", {"sort": ["id:desc"], "offset": offset, "limit": limit}
)
return list(map(Info.from_dict, response["hits"]))
async def get_random_info(self) -> Info:
total = await self.get_total()
response = await self.index.get_documents(
{"offset": randrange(total), "limit": 1}
)
d = cast(HitomiInfoJSON, response[0])
return Info.from_dict(d)
async def search(
self,
tags: list[str],
offset: int = 0,
limit: int = 15,
) -> tuple[list[Info], int]:
title, tags = self.parse_query(tags)
response = await self.index.search(
title,
{"sort": ["id:desc"], "filter": tags, "offset": offset, "limit": limit},
)
return list(map(Info.from_dict, response["hits"])), response["nbHits"]

148
heliotrope/database/odm/__init__.py Normal file
View file

@ -0,0 +1,148 @@
# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false
# Motor 라이브러리는 유형 주석이 적용 되어있지 않기때문에 여러 타입 관련 문제를 무시합니다.
# The Motor library ignores many type-related issues because type annotations are not applied.
from typing import Any, Optional, cast
from motor.core import AgnosticClient, AgnosticCollection # type: ignore
from motor.motor_asyncio import AsyncIOMotorClient # type: ignore
from heliotrope.abc.database import AbstractInfoDatabase
from heliotrope.domain.info import Info
from heliotrope.types import HitomiInfoJSON
class ODM(AbstractInfoDatabase):
def __init__(
self, client: Any, is_atlas: bool = False, use_atlas_search: bool = False
) -> None:
self.client: AgnosticClient = client
self.collection: AgnosticCollection = self.client.hitomi.info
self.is_atlas = is_atlas
self.use_atlas_search = use_atlas_search
def close(self) -> None:
self.client.close()
@classmethod
def setup(cls, mongo_db_url: str, use_atlas_search: bool = False) -> "ODM":
is_atlas = "mongodb.net" in mongo_db_url
return cls(AsyncIOMotorClient(mongo_db_url), is_atlas, use_atlas_search)
async def get_all_index(self) -> list[int]:
ids: list[int] = []
id: int
async for id in self.collection.find({}, {"id": 1}):
ids.append(id)
return ids
async def add_info(self, info: Info) -> None:
await self.collection.insert_one(info.to_dict())
async def get_info(self, id: int) -> Optional[Info]:
info_json = cast(
Optional[HitomiInfoJSON],
await self.collection.find_one({"id": id}, {"_id": 0}),
)
if info_json:
return Info.from_dict(info_json)
return None
async def get_info_list(self, offset: int = 0, limit: int = 15) -> list[Info]:
offset = offset * limit
info_jsons = cast(
list[HitomiInfoJSON],
await self.collection.find({}, {"_id": 0})
.sort("id", -1)
.skip(offset)
.limit(limit)
.to_list(15),
)
return [Info.from_dict(json_info) for json_info in info_jsons]
async def get_random_info(self) -> Info:
info_json = cast(
HitomiInfoJSON,
await self.collection.aggregate(
[
{"$sample": {"size": 1}},
{"$project": {"_id": 0}},
]
).next(),
)
return Info.from_dict(info_json)
def parse_query(self, querys: list[str]) -> tuple[str, dict[str, Any]]:
# Tags are received in the following format: female:big_breasts
# If it is not in the following format, it is regarded as a title.
# 태그는 다음과 같은 형식으로 받습니다.: female:big_breasts
# 만약 다음과 같은 형식이 아니라면 제목으로 간주합니다.
query_dict: dict[str, Any] = {}
title = ""
for query in querys:
if any(info_tag in query for info_tag in self.info_tags):
splited = query.split(":")
query_dict.update({splited[0]: splited[1]})
elif any(
gender_common_tag in query
for gender_common_tag in self.gender_common_tags
):
query_dict.update({"tags": query})
else:
title = query
return title, query_dict
def make_pipeline(
self, title: str, query: dict[str, Any], offset: int = 0, limit: int = 15
) -> list[dict[str, Any]]:
pipeline: list[dict[str, Any]] = [
{"$match": query},
{
"$group": {
"_id": 0,
"count": {"$sum": 1},
"list": {"$push": "$$ROOT"},
}
},
{"$skip": offset},
{"$limit": limit},
{"$project": {"_id": 0, "list": {"_id": 0}}},
]
if title:
if self.is_atlas and self.use_atlas_search:
pipeline.insert(
0,
{
"$search": {
"compound": {
"must": [{"text": {"query": title, "path": "title"}}]
}
}
},
)
else:
pipeline[0]["$match"].update({"$text": {"$search": title}})
return pipeline
async def search(
self, querys: list[str], offset: int = 0, limit: int = 15
) -> tuple[list[Info], int]:
offset = offset * limit
title, query = self.parse_query(querys)
pipeline = self.make_pipeline(title, query, offset, limit)
try:
results: dict[str, Any] = await self.collection.aggregate(pipeline).next()
except StopAsyncIteration:
results = {"list": [], "count": 0}
return [
Info.from_dict(cast(HitomiInfoJSON, result)) for result in results["list"]
], results["count"]

47
heliotrope/database/orm/__init__.py
View file

@ -27,7 +27,7 @@ from typing import Any, Optional
from sanic.log import logger
from sqlalchemy.ext.asyncio.engine import AsyncEngine, create_async_engine
from sqlalchemy.ext.asyncio.session import AsyncSession
from sqlalchemy.orm import mapper, relationship, selectinload
from sqlalchemy.orm import relationship, selectinload
from sqlalchemy.orm.exc import UnmappedClassError
from sqlalchemy.orm.mapper import class_mapper
from sqlalchemy.orm.session import sessionmaker
@ -35,12 +35,8 @@ from sqlalchemy.sql.expression import select
from heliotrope.abc.database import AbstractGalleryinfoDatabase
from heliotrope.database.orm.base import mapper_registry
from heliotrope.database.orm.table.file import file_table
from heliotrope.database.orm.table.gallleryinfo import galleryinfo_table
from heliotrope.database.orm.table.tag import tag_table
from heliotrope.domain.file import File
from heliotrope.domain.galleryinfo import Galleryinfo
from heliotrope.domain.tag import Tag
from heliotrope.database.orm.table import *
from heliotrope.domain import *
from heliotrope.utils import is_the_first_process
_base_model_session_ctx: ContextVar[AsyncSession] = ContextVar("session")
@ -67,13 +63,30 @@ class ORM(AbstractGalleryinfoDatabase):
@staticmethod
def mapping() -> None:
mapper(
mapper_registry.map_imperatively(
Galleryinfo,
galleryinfo_table,
properties={"files": relationship(File), "tags": relationship(Tag)},
properties={
"files": relationship(File),
"tags": relationship(Tag),
"artists": relationship(Artist),
"characters": relationship(Character),
"groups": relationship(Group),
"parodys": relationship(Parody),
"scene_indexes": relationship(SceneIndex),
"languages": relationship(Language),
"related": relationship(Related),
},
)
mapper(Tag, tag_table)
mapper(File, file_table)
mapper_registry.map_imperatively(Tag, tag_table)
mapper_registry.map_imperatively(File, file_table)
mapper_registry.map_imperatively(Artist, artist_table)
mapper_registry.map_imperatively(Character, character_table)
mapper_registry.map_imperatively(Group, group_table)
mapper_registry.map_imperatively(Parody, parody_table)
mapper_registry.map_imperatively(SceneIndex, scene_index_table)
mapper_registry.map_imperatively(Related, related_table)
mapper_registry.map_imperatively(Language, language_table)
return None
@classmethod
@ -106,7 +119,17 @@ class ORM(AbstractGalleryinfoDatabase):
r = await manager.session.get(
Galleryinfo,
id,
[selectinload(Galleryinfo.files), selectinload(Galleryinfo.tags)],
[
selectinload(Galleryinfo.files),
selectinload(Galleryinfo.tags),
selectinload(Galleryinfo.artists),
selectinload(Galleryinfo.characters),
selectinload(Galleryinfo.groups),
selectinload(Galleryinfo.parodys),
selectinload(Galleryinfo.scene_indexes),
selectinload(Galleryinfo.languages),
selectinload(Galleryinfo.related),
],
)
if r:
return r

23
heliotrope/database/orm/table/__init__.py
View file

@ -21,3 +21,26 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from heliotrope.database.orm.table.artist import artist_table
from heliotrope.database.orm.table.character import character_table
from heliotrope.database.orm.table.file import file_table
from heliotrope.database.orm.table.gallleryinfo import galleryinfo_table
from heliotrope.database.orm.table.group import group_table
from heliotrope.database.orm.table.language import language_table
from heliotrope.database.orm.table.parody import parody_table
from heliotrope.database.orm.table.related import related_table
from heliotrope.database.orm.table.scene_index import scene_index_table
from heliotrope.database.orm.table.tag import tag_table
__all__ = [
"artist_table",
"character_table",
"file_table",
"galleryinfo_table",
"group_table",
"language_table",
"parody_table",
"related_table",
"scene_index_table",
"tag_table",
]

36
heliotrope/database/orm/table/artist.py Normal file
View file

@ -0,0 +1,36 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer, String
from heliotrope.database.orm.base import mapper_registry
artist_table = Table(
"artist",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleyinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("artist", String, nullable=False),
Column("url", String, nullable=False),
)

36
heliotrope/database/orm/table/character.py Normal file
View file

@ -0,0 +1,36 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer, String
from heliotrope.database.orm.base import mapper_registry
character_table = Table(
"character",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("character", String, nullable=False),
Column("url", String, nullable=False),
)

4
heliotrope/database/orm/table/file.py
View file

@ -30,12 +30,12 @@ file_table = Table(
"file",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("index_id", Integer, ForeignKey("galleryinfo.id")),
Column("galleyinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("name", String, nullable=False),
Column("width", Integer, nullable=False),
Column("height", Integer, nullable=False),
Column("hash", String(64), nullable=False),
Column("haswebp", Integer, nullable=False),
Column("hasavifsmalltn", Integer),
Column("hasavif", Integer),
Column("hasavif", Integer, nullable=False),
)

10
heliotrope/database/orm/table/gallleryinfo.py
View file

@ -30,10 +30,16 @@ galleryinfo_table = Table(
"galleryinfo",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column("type", String, nullable=False),
# title
Column("title", String, nullable=False),
Column("japanese_title", String),
Column("language", String, nullable=False),
# video
Column("video", String),
Column("videofilename", String),
# language
Column("language_url", String),
Column("language_localname", String, nullable=False),
Column("type", String, nullable=False),
Column("language", String, nullable=False),
Column("date", String, nullable=False),
)

37
heliotrope/database/orm/table/group.py Normal file
View file

@ -0,0 +1,37 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy import ForeignKey, String
from sqlalchemy.sql.schema import Column, Table
from sqlalchemy.sql.sqltypes import Integer
from heliotrope.database.orm.base import mapper_registry
group_table = Table(
"group",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("group", String, nullable=False),
Column("url", String, nullable=False),
)

38
heliotrope/database/orm/table/language.py Normal file
View file

@ -0,0 +1,38 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer, String
from heliotrope.database.orm.base import mapper_registry
language_table = Table(
"language",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("url", String, nullable=False),
Column("name", String, nullable=False),
Column("galleryid", String, nullable=False),
Column("language_localname", String, nullable=False),
)

36
heliotrope/database/orm/table/parody.py Normal file
View file

@ -0,0 +1,36 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer, String
from heliotrope.database.orm.base import mapper_registry
parody_table = Table(
"parody",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("parody", String, nullable=False),
Column("url", String, nullable=False),
)

35
heliotrope/database/orm/table/related.py Normal file
View file

@ -0,0 +1,35 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer
from heliotrope.database.orm.base import mapper_registry
related_table = Table(
"related",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("related_id", Integer),
)

35
heliotrope/database/orm/table/scene_index.py Normal file
View file

@ -0,0 +1,35 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from sqlalchemy.sql.schema import Column, ForeignKey, Table
from sqlalchemy.sql.sqltypes import Integer
from heliotrope.database.orm.base import mapper_registry
scene_index_table = Table(
"scene_index",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("galleryinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("scene_index", Integer),
)

2
heliotrope/database/orm/table/tag.py
View file

@ -30,7 +30,7 @@ tag_table = Table(
"tag",
mapper_registry.metadata,
Column("id", Integer, primary_key=True, autoincrement=True),
Column("index_id", Integer, ForeignKey("galleryinfo.id")),
Column("galleyinfo_id", Integer, ForeignKey("galleryinfo.id")),
Column("male", String(1)),
Column("female", String(1)),
Column("tag", String, nullable=False),

25
heliotrope/domain/__init__.py
View file

@ -21,3 +21,28 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from heliotrope.domain.artist import Artist
from heliotrope.domain.character import Character
from heliotrope.domain.file import File
from heliotrope.domain.galleryinfo import Galleryinfo
from heliotrope.domain.group import Group
from heliotrope.domain.info import Info
from heliotrope.domain.language import Language
from heliotrope.domain.parody import Parody
from heliotrope.domain.related import Related
from heliotrope.domain.scene_index import SceneIndex
from heliotrope.domain.tag import Tag
__all__ = [
"Artist",
"Character",
"File",
"Galleryinfo",
"Group",
"Info",
"Language",
"Parody",
"Tag",
"SceneIndex",
"Related",
]

48
heliotrope/domain/artist.py Normal file
View file

@ -0,0 +1,48 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
from heliotrope.types import HitomiArtistsJSON
@dataclass
class Artist:
galleryinfo_id: int
artist: str
url: str
id: int = field(init=False)
def to_dict(self) -> HitomiArtistsJSON:
return HitomiArtistsJSON(
artist=self.artist,
url=self.url,
)
@classmethod
def from_dict(cls, galleryinfo_id: int, d: HitomiArtistsJSON) -> "Artist":
return cls(
galleryinfo_id=galleryinfo_id,
artist=d["artist"],
url=d["url"],
)

48
heliotrope/domain/character.py Normal file
View file

@ -0,0 +1,48 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
from heliotrope.types import HitomiCharatersJSON
@dataclass
class Character:
galleryinfo_id: int
character: str
url: str
id: int = field(init=False)
def to_dict(self) -> HitomiCharatersJSON:
return HitomiCharatersJSON(
character=self.character,
url=self.url,
)
@classmethod
def from_dict(cls, galleryinfo_id: int, d: HitomiCharatersJSON) -> "Character":
return cls(
galleryinfo_id=galleryinfo_id,
character=d["character"],
url=d["url"],
)

18
heliotrope/domain/file.py
View file

@ -21,7 +21,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Literal, Optional
from heliotrope.types import HitomiFileJSON
@ -29,15 +29,15 @@ from heliotrope.types import HitomiFileJSON
@dataclass
class File:
index_id: int
galleryinfo_id: int
name: str
width: int
height: int
hash: str
haswebp: Literal[0, 1]
hasavif: Literal[0, 1]
hasavifsmalltn: Optional[Literal[1]] = None
hasavif: Optional[Literal[1]] = None
id: Optional[int] = None
id: int = field(init=False)
def to_dict(self) -> HitomiFileJSON:
hitomi_file_json = HitomiFileJSON(
@ -45,25 +45,23 @@ class File:
hash=self.hash,
haswebp=self.haswebp,
name=self.name,
hasavif=self.hasavif,
height=self.height,
)
if self.hasavif is not None:
hitomi_file_json["hasavif"] = self.hasavif
if self.hasavifsmalltn is not None:
hitomi_file_json["hasavifsmalltn"] = self.hasavifsmalltn
return hitomi_file_json
@classmethod
def from_dict(cls, index_id: int, d: HitomiFileJSON) -> "File":
def from_dict(cls, galleryinfo_id: int, d: HitomiFileJSON) -> "File":
return cls(
index_id=index_id,
galleryinfo_id=galleryinfo_id,
name=d["name"],
width=d["width"],
height=d["height"],
hash=d["hash"],
haswebp=d["haswebp"],
hasavif=d["hasavif"],
hasavifsmalltn=d.get("hasavifsmalltn"),
hasavif=d.get("hasavif"),
)

83
heliotrope/domain/galleryinfo.py
View file

@ -24,7 +24,14 @@ SOFTWARE.
from dataclasses import dataclass, field
from typing import Optional
from heliotrope.domain.artist import Artist
from heliotrope.domain.character import Character
from heliotrope.domain.file import File
from heliotrope.domain.group import Group
from heliotrope.domain.language import Language
from heliotrope.domain.parody import Parody
from heliotrope.domain.related import Related
from heliotrope.domain.scene_index import SceneIndex
from heliotrope.domain.tag import Tag
from heliotrope.types import HitomiGalleryinfoJSON
@ -32,26 +39,58 @@ from heliotrope.types import HitomiGalleryinfoJSON
@dataclass
class Galleryinfo:
id: int
title: str
japanese_title: Optional[str]
language: Optional[str]
language_localname: str
type: str
date: str
# title
title: str
japanese_title: Optional[str] = None
# video
video: Optional[str] = None
videofilename: Optional[str] = None
# language
language_url: Optional[str] = None
language_localname: Optional[str] = None
language: Optional[str] = None
languages: list[Language] = field(default_factory=list)
# tags
files: list[File] = field(default_factory=list)
tags: list[Tag] = field(default_factory=list)
tags: Optional[list[Tag]] = None
artists: Optional[list[Artist]] = None
characters: Optional[list[Character]] = None
groups: Optional[list[Group]] = None
parodys: Optional[list[Parody]] = None
scene_indexes: list[SceneIndex] = field(default_factory=list)
related: list[Related] = field(default_factory=list)
def to_dict(self) -> HitomiGalleryinfoJSON:
return HitomiGalleryinfoJSON(
title=self.title,
id=self.id,
date=self.date,
type=self.type,
date=self.date,
title=self.title,
japanese_title=self.japanese_title,
language=self.language,
files=[file.to_dict() for file in self.files],
video=self.video,
videofilename=self.videofilename,
language_url=self.language_url,
language_localname=self.language_localname,
tags=[tag.to_dict() for tag in self.tags],
language=self.language,
languages=[language.to_dict() for language in self.languages],
files=[file.to_dict() for file in self.files],
related=[related.to_id() for related in self.related],
scene_indexes=[
scene_index.to_index() for scene_index in self.scene_indexes
],
tags=[tag.to_dict() for tag in self.tags] if self.tags else None,
artists=[artist.to_dict() for artist in self.artists]
if self.artists
else None,
characters=[character.to_dict() for character in self.characters]
if self.characters
else None,
groups=[group.to_dict() for group in self.groups] if self.groups else None,
parodys=[parody.to_dict() for parody in self.parodys]
if self.parodys
else None,
)
@classmethod
@ -65,6 +104,28 @@ class Galleryinfo:
language_localname=d["language_localname"],
type=d["type"],
date=d["date"],
languages=[
Language.from_dict(int_id, language) for language in d["languages"]
],
related=[Related.from_dict(int_id, related) for related in d["related"]],
files=[File.from_dict(int_id, file) for file in d["files"]],
tags=[Tag.from_dict(int_id, tag) for tag in d["tags"]],
scene_indexes=[
SceneIndex.from_dict(int_id, scene_index)
for scene_index in d["scene_indexes"]
],
tags=[Tag.from_dict(int_id, tag) for tag in d["tags"]] if d["tags"] else [],
artists=[Artist.from_dict(int_id, artist) for artist in d["artists"]]
if d["artists"]
else [],
characters=[
Character.from_dict(int_id, character) for character in d["characters"]
]
if d["characters"]
else [],
groups=[Group.from_dict(int_id, group) for group in d["groups"]]
if d["groups"]
else [],
parodys=[Parody.from_dict(int_id, parody) for parody in d["parodys"]]
if d["parodys"]
else [],
)

48
heliotrope/domain/group.py Normal file
View file

@ -0,0 +1,48 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
from heliotrope.types import HitomiGroupsJSON
@dataclass
class Group:
galleryinfo_id: int
group: str
url: str
id: int = field(init=False)
def to_dict(self) -> HitomiGroupsJSON:
return HitomiGroupsJSON(
group=self.group,
url=self.url,
)
@classmethod
def from_dict(cls, galleryinfo_id: int, d: HitomiGroupsJSON) -> "Group":
return cls(
galleryinfo_id=galleryinfo_id,
group=d["group"],
url=d["url"],
)

89
heliotrope/domain/info.py
View file

@ -22,45 +22,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass
from typing import Optional
from typing import Mapping, Optional
from bs4.element import Tag
from heliotrope.parser import Parser
from heliotrope.domain.file import File
from heliotrope.domain.galleryinfo import Galleryinfo
from heliotrope.domain.tag import Tag
from heliotrope.types import HitomiInfoJSON
def from_element(element: Tag) -> str:
return element.text.strip().replace(" ", "_")
def parse_tags_dict_list(tags_dict_list: list[Mapping[str, object]]) -> list[str]:
return [str(v) for tags in tags_dict_list for k, v in tags.items() if k != "url"]
def from_elements(elements: list[Tag]) -> list[str]:
return [from_element(element) for element in elements]
def tags_replacer(values: list[str]) -> list[str]:
replaced: list[str] = []
for value in values:
if "♀" in value:
removed_icon = value.replace("_♀", "")
value = f"female:{removed_icon}"
elif "♂" in value:
removed_icon = value.replace("_♂", "")
value = f"male:{removed_icon}"
else:
value = f"tag:{value}"
replaced.append(value)
return replaced
def parse_male_female_tag(tag: Tag) -> str:
tag_name = tag.tag.replace(" ", "_")
if tag.male:
return f"male:{tag_name}"
if tag.female:
return f"female:{tag_name}"
return f"tag:{tag_name}"
@dataclass
class Info:
id: int
title: str
thumbnail: str
thumbnail: File
artist: list[str]
group: list[str]
type: str
@ -71,21 +58,37 @@ class Info:
date: str
@classmethod
def from_parser(cls, id: int, parser: Parser) -> "Info":
def from_galleryinfo(cls, galleryinfo: Galleryinfo) -> "Info":
return cls(
id=id,
title=parser.title_element.text,
thumbnail=parser.thumbnail_element.attrs["src"],
artist=from_elements(parser.artist_elements),
group=from_elements(parser.group_elements),
type=from_element(parser.type_element),
language=from_element(parser.language_element)
if parser.language_element
else None,
series=from_elements(parser.series_elements),
character=from_elements(parser.character_elements),
tags=tags_replacer(from_elements(parser.tags_elements)),
date=parser.date_element.text,
id=galleryinfo.id,
title=galleryinfo.title,
thumbnail=galleryinfo.files[0],
artist=parse_tags_dict_list(
[artist.to_dict() for artist in galleryinfo.artists]
)
if galleryinfo.artists
else [],
group=parse_tags_dict_list(
[group.to_dict() for group in galleryinfo.groups]
)
if galleryinfo.groups
else [],
type=galleryinfo.type,
language=galleryinfo.language,
series=parse_tags_dict_list(
[parody.to_dict() for parody in galleryinfo.parodys]
)
if galleryinfo.parodys
else [],
character=parse_tags_dict_list(
[character.to_dict() for character in galleryinfo.characters]
)
if galleryinfo.characters
else [],
tags=[parse_male_female_tag(tag) for tag in galleryinfo.tags]
if galleryinfo.tags
else [],
date=galleryinfo.date,
)
@classmethod
@ -93,7 +96,7 @@ class Info:
return cls(
id=int(d["id"]),
title=d["title"],
thumbnail=d["thumbnail"],
thumbnail=File.from_dict(int(d["id"]), d["thumbnail"]),
artist=d["artist"],
group=d["group"],
type=d["type"],
@ -108,7 +111,7 @@ class Info:
return HitomiInfoJSON(
id=self.id,
title=self.title,
thumbnail=self.thumbnail,
thumbnail=self.thumbnail.to_dict(),
artist=self.artist,
group=self.group,
type=self.type,

54
heliotrope/domain/language.py Normal file
View file

@ -0,0 +1,54 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
from heliotrope.types import HitomiLanguagesJSON
@dataclass
class Language:
galleryinfo_id: int
url: str
name: str
galleryid: str
language_localname: str
id: int = field(init=False)
def to_dict(self) -> HitomiLanguagesJSON:
return HitomiLanguagesJSON(
url=self.url,
name=self.name,
galleryid=self.galleryid,
language_localname=self.language_localname,
)
@classmethod
def from_dict(cls, galleryinfo_id: int, d: HitomiLanguagesJSON) -> "Language":
return cls(
galleryinfo_id=galleryinfo_id,
url=d["url"],
name=d["name"],
galleryid=d["galleryid"],
language_localname=d["language_localname"],
)

48
heliotrope/domain/parody.py Normal file
View file

@ -0,0 +1,48 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
from heliotrope.types import HitomiParodysJSON
@dataclass
class Parody:
galleryinfo_id: int
parody: str
url: str
id: int = field(init=False)
def to_dict(self) -> HitomiParodysJSON:
return HitomiParodysJSON(
parody=self.parody,
url=self.url,
)
@classmethod
def from_dict(cls, galleryinfo_id: int, d: HitomiParodysJSON) -> "Parody":
return cls(
galleryinfo_id=galleryinfo_id,
parody=d["parody"],
url=d["url"],
)

41
heliotrope/domain/related.py Normal file
View file

@ -0,0 +1,41 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
@dataclass
class Related:
galleryinfo_id: int
id: int = field(init=False)
related_id: int
def to_id(self) -> int:
return self.related_id
@classmethod
def from_dict(cls, galleryinfo_id: int, id: int) -> "Related":
return cls(
galleryinfo_id=galleryinfo_id,
related_id=id,
)

41
heliotrope/domain/scene_index.py Normal file
View file

@ -0,0 +1,41 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass, field
@dataclass
class SceneIndex:
galleryinfo_id: int
id: int = field(init=False)
scene_index: int
def to_index(self) -> int:
return self.scene_index
@classmethod
def from_dict(cls, galleryinfo_id: int, scene_index: int) -> "SceneIndex":
return cls(
galleryinfo_id=galleryinfo_id,
scene_index=scene_index,
)

14
heliotrope/domain/tag.py
View file

@ -21,7 +21,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Literal, Optional, cast
from heliotrope.types import HitomiTagJSON
@ -35,12 +35,12 @@ class Tag:
Literal["", "1"], str(self.female) if self.female else self.female
)
index_id: int
male: Optional[Literal["", "1"]]
female: Optional[Literal["", "1"]]
galleryinfo_id: int
male: Optional[Literal["", "1", 1]]
female: Optional[Literal["", "1", 1]]
tag: str
url: str
id: Optional[int] = None
id: int = field(init=False)
def to_dict(self) -> HitomiTagJSON:
hitomi_tag_json = HitomiTagJSON(url=self.url, tag=self.tag)
@ -54,9 +54,9 @@ class Tag:
return hitomi_tag_json
@classmethod
def from_dict(cls, index_id: int, d: HitomiTagJSON) -> "Tag":
def from_dict(cls, galleryinfo_id: int, d: HitomiTagJSON) -> "Tag":
return cls(
index_id=index_id,
galleryinfo_id=galleryinfo_id,
male=d.get("male"),
female=d.get("female"),
tag=d["tag"],

10
heliotrope/interpreter/__init__.py
View file

@ -98,7 +98,7 @@ class CommonJS:
def parse_gg_js(self, code: str) -> str:
lines = StringIO(code).readlines()
return "".join([line for line in lines if "return 4" not in line])
return "".join([line for line in lines if "if (!" not in line])
def update_js_code(self, common_js_code: str, gg_js_code: str) -> None:
self.common_js_code = common_js_code
@ -123,10 +123,16 @@ class CommonJS:
),
)
async def get_thumbnail(self, galleryid: int, image: HitomiFileJSON) -> str:
return cast(
str,
await to_thread(self.interpreter.getThumbnail, galleryid, image),
)
async def image_urls(
self, galleryid: int, images: list[HitomiFileJSON], no_webp: bool
) -> dict[str, str]:
return cast(
dict[str, str],
await to_thread(self.interpreter.image_urls, galleryid, images, no_webp),
await to_thread(self.interpreter.imageUrls, galleryid, images, no_webp),
)

8
heliotrope/interpreter/function.js
View file

@ -2,12 +2,16 @@
// Leave all the work to the interpreter.
// 계속해서 인터프리터의 함수를 호출을할경우 매우 느려집니다.
// 인터프리터에 모든작업을 맡깁니다.
function image_urls(galleryid, images, no_webp) {
function imageUrls(galleryid, images, no_webp) {
return images.map(function (image) {
var webp = null
if (image.hash && image.haswebp && !no_webp) {
webp = 'webp'
}
return { 'name': image.name, 'url': url_from_url_from_hash(galleryid, image, webp) }
return { 'name': image.name, 'url': url_from_url_from_hash(galleryid, image, webp, undefined, "a") }
})
}
// See https://ltn.hitomi.la/gallery.js
function getThumbnail(galleryid, image) {
return url_from_url_from_hash(galleryid, image, 'webpbigtn', 'webp', 'tn')
}

124
heliotrope/parser.py
View file

@ -1,124 +0,0 @@
"""
MIT License
Copyright (c) 2021 SaidBySolo
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from typing import Optional
from bs4 import BeautifulSoup
from bs4.element import NavigableString, Tag
class BaseParser:
HITOMI_TYPE_MAPPING = {
"manga": "manga",
"doujinshi": "dj",
"cg": "acg",
"gamecg": "cg",
"anime": "anime",
}
def __init__(self, html: str, hitomi_type: str) -> None:
self.__html = html
self.__hitomi_type = hitomi_type
@property
def soup_type(self) -> str:
return self.HITOMI_TYPE_MAPPING[self.__hitomi_type]
@property
def soup(self) -> BeautifulSoup:
return BeautifulSoup(self.__html, "lxml")
@property
def gallery_element(self) -> Tag:
gallery_element = self.soup.find(
"div", {"class": f"gallery {self.soup_type}-gallery"}
)
assert isinstance(gallery_element, Tag)
return gallery_element
@property
def infos(self) -> list[Tag]:
galleryinfo = self.gallery_element.find("div", {"class": "gallery-info"})
assert isinstance(galleryinfo, Tag)
return galleryinfo.find_all("tr")
class Parser:
def __init__(self, html: str, hitomi_type: str) -> None:
self.base_parser = BaseParser(html, hitomi_type)
@property
def title_element(self) -> Tag:
title_element = self.base_parser.gallery_element.find("h1")
assert isinstance(title_element, Tag)
title = title_element.find("a")
assert isinstance(title, Tag)
return title
@property
def thumbnail_element(self) -> Tag:
picture_element = self.base_parser.soup.find("picture")
assert isinstance(picture_element, Tag)
img_element = picture_element.find("img")
assert isinstance(img_element, Tag)
return img_element
@property
def artist_elements(self) -> list[Tag]:
artist_element = self.base_parser.soup.find("h2")
assert isinstance(artist_element, Tag)
return artist_element.find_all("a")
@property
def group_elements(self) -> list[Tag]:
return self.base_parser.infos[0].find_all("a")
@property
def type_element(self) -> Tag:
type_element = self.base_parser.infos[1].find("a")
assert isinstance(type_element, Tag)
return type_element
@property
def language_element(self) -> Optional[Tag]:
language_element = self.base_parser.infos[2].find("a")
assert not isinstance(language_element, NavigableString)
return language_element
@property
def series_elements(self) -> list[Tag]:
return self.base_parser.infos[3].find_all("a")
@property
def character_elements(self) -> list[Tag]:
return self.base_parser.infos[4].find_all("a")
@property
def tags_elements(self) -> list[Tag]:
return self.base_parser.infos[5].find_all("a")
@property
def date_element(self) -> Tag:
date_elemment = self.base_parser.soup.find("span", class_="date")
assert isinstance(date_elemment, Tag)
return date_elemment

28
heliotrope/request/hitomi.py
View file

@ -23,7 +23,7 @@ SOFTWARE.
"""
from json import loads
from struct import unpack
from typing import Any, Optional
from typing import Any, Callable, Optional
from aiohttp.client import ClientSession
from bs4 import BeautifulSoup
@ -31,10 +31,9 @@ from bs4.element import Tag
from sanic.log import logger
from yarl import URL
from heliotrope.domain.galleryinfo import Galleryinfo
from heliotrope.domain.info import Info
from heliotrope.parser import Parser
from heliotrope.domain import Galleryinfo, Info
from heliotrope.request.base import BaseRequest
from heliotrope.types import HitomiFileJSON
class HitomiRequest:
@ -105,27 +104,6 @@ class HitomiRequest:
return Galleryinfo.from_dict(js_to_json)
async def get_info_parser(self, id: int) -> Optional[Parser]:
response = await self.get_redirect_url(id)
if not response:
return None
url, hitomi_type = response
html = await self.request.get(url, "text")
if "Redirect" in html.body:
return None
return Parser(html.body, hitomi_type)
async def get_info(self, id: int) -> Optional[Info]:
parser = await self.get_info_parser(id)
if not parser:
return None
return Info.from_parser(id, parser)
# NOTE: See https://ltn.hitomi.la/galleryblock.js
# 참고: https://ltn.hitomi.la/galleryblock.js
async def fetch_index(

5
heliotrope/rest/api/hitomi/image.py
View file

@ -21,9 +21,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from asyncio.tasks import gather
from typing import Any, Coroutine
from sanic.blueprints import Blueprint
from sanic.exceptions import NotFound
from sanic.response import HTTPResponse, json
@ -49,7 +46,7 @@ class HitomiImageView(HTTPMethodView):
raise NotFound
files = await request.app.ctx.common_js.image_urls(
id, list(map(lambda f: f.to_dict(), galleryinfo.files)), True
id, list(map(lambda f: f.to_dict(), galleryinfo.files)), False
)
return json(

24
heliotrope/rest/api/hitomi/info.py
View file

@ -21,12 +21,15 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from typing import Any
from sanic.blueprints import Blueprint
from sanic.exceptions import NotFound
from sanic.response import HTTPResponse, json
from sanic.views import HTTPMethodView
from sanic_ext.extensions.openapi import openapi # type: ignore
from heliotrope.domain import Info
from heliotrope.sanic import HeliotropeRequest
hitomi_info = Blueprint("hitomi_info", url_prefix="/info")
@ -37,13 +40,24 @@ class HitomiInfoView(HTTPMethodView):
@openapi.summary("Get hitomi info") # type: ignore
@openapi.parameter(name="id", location="path", schema=int) # type: ignore
async def get(self, request: HeliotropeRequest, id: int) -> HTTPResponse:
if info := await request.app.ctx.meilisearch.get_info(id):
return json({"status": 200, **info.to_dict()})
info = await request.app.ctx.odm.get_info(id)
if requested_info := await request.app.ctx.hitomi_request.get_info(id):
return json({"status": 200, **requested_info.to_dict()})
if not info:
if galleryinfo := await request.app.ctx.hitomi_request.get_galleryinfo(id):
info = Info.from_galleryinfo(galleryinfo)
else:
raise NotFound
raise NotFound
info_dict = info.to_dict()
thumbnail = await request.app.ctx.common_js.get_thumbnail(
id, info_dict["thumbnail"]
)
res: dict[str, Any] = {
"status": 200,
**info_dict,
}
res["thumbnail"] = thumbnail
return json(res)
hitomi_info.add_route(HitomiInfoView.as_view(), "/<id:int>")

2
heliotrope/rest/api/hitomi/list.py
View file

@ -44,7 +44,7 @@ class HitomiListView(HTTPMethodView):
if start_at_zero < 0 or total < start_at_zero:
raise InvalidUsage
info_list = await request.app.ctx.meilisearch.get_info_list(start_at_zero)
info_list = await request.app.ctx.odm.get_info_list(start_at_zero)
return json(
{

2
heliotrope/rest/api/hitomi/random.py
View file

@ -35,7 +35,7 @@ class HitomiRandomView(HTTPMethodView):
@openapi.summary("Get random result in hitomi") # type: ignore
@openapi.tag("hitomi") # type: ignore
async def get(self, request: HeliotropeRequest) -> HTTPResponse:
info = await request.app.ctx.meilisearch.get_random_info()
info = await request.app.ctx.odm.get_random_info()
return json({"status": 200, **info.to_dict()})

2
heliotrope/rest/api/hitomi/search.py
View file

@ -53,7 +53,7 @@ class HitomiSearchView(HTTPMethodView):
else 0
)
if (query := request.json.get("query")) and query:
results, count = await request.app.ctx.meilisearch.search(query, offset)
results, count = await request.app.ctx.odm.search(query, offset)
return json(
{
"status": 200,

4
heliotrope/sanic.py
View file

@ -28,7 +28,7 @@ from sanic.app import Sanic
from sanic.request import Request
from heliotrope.config import HeliotropeConfig
from heliotrope.database.meilisearch import MeiliSearch
from heliotrope.database.odm import ODM
from heliotrope.database.orm import ORM
from heliotrope.interpreter import CommonJS
from heliotrope.request.base import BaseRequest
@ -37,7 +37,7 @@ from heliotrope.request.hitomi import HitomiRequest
class HeliotropeContext(SimpleNamespace):
orm: ORM
meilisearch: MeiliSearch
odm: ODM
request: BaseRequest
hitomi_request: HitomiRequest
common_js: CommonJS

8
heliotrope/server.py
View file

@ -28,7 +28,7 @@ from sentry_sdk.integrations.sanic import SanicIntegration
from heliotrope import __version__
from heliotrope.config import HeliotropeConfig
from heliotrope.database.meilisearch import MeiliSearch
from heliotrope.database.odm import ODM
from heliotrope.database.orm import ORM
from heliotrope.interpreter import CommonJS
from heliotrope.request.base import BaseRequest
@ -43,8 +43,8 @@ from heliotrope.utils import is_the_first_process
async def startup(heliotrope: Heliotrope, loop: AbstractEventLoop) -> None:
# DB and http setup
heliotrope.ctx.meilisearch = await MeiliSearch.setup(
heliotrope.config.INFO_DB_URL, heliotrope.config.INFO_DB_API_KEY
heliotrope.ctx.odm = ODM.setup(
heliotrope.config.INFO_DB_URL,
)
heliotrope.ctx.orm = await ORM.setup(heliotrope.config.GALLERYINFO_DB_URL)
heliotrope.ctx.request = await BaseRequest.setup()
@ -72,9 +72,9 @@ async def startup(heliotrope: Heliotrope, loop: AbstractEventLoop) -> None:
async def closeup(heliotrope: Heliotrope, loop: AbstractEventLoop) -> None:
# Close session
await heliotrope.ctx.meilisearch.close()
await heliotrope.ctx.request.close()
await heliotrope.ctx.hitomi_request.close()
heliotrope.ctx.odm.close()
# Close task
heliotrope.shutdown_tasks()

27
heliotrope/tasks/mirroring.py
View file

@ -29,6 +29,7 @@ from sanic.log import logger
from heliotrope.abc.database import AbstractGalleryinfoDatabase, AbstractInfoDatabase
from heliotrope.abc.task import AbstractTask
from heliotrope.domain.info import Info
from heliotrope.request.hitomi import HitomiRequest
from heliotrope.sanic import Heliotrope
from heliotrope.types import SetupTask
@ -48,7 +49,7 @@ class MirroringTask(AbstractTask):
@classmethod
def setup(cls, app: Heliotrope, delay: float) -> SetupTask:
logger.debug(f"Setting up {cls.__name__}.")
instance = cls(app.ctx.hitomi_request, app.ctx.orm, app.ctx.meilisearch)
instance = cls(app.ctx.hitomi_request, app.ctx.orm, app.ctx.odm)
return create_task(instance.start(delay))
async def compare_index_list(self) -> list[int]:
@ -69,26 +70,22 @@ class MirroringTask(AbstractTask):
if galleryinfo := await self.request.get_galleryinfo(index):
logger.debug(f"{index} can get galleryinfo from hitomi.la.")
await self.galleryinfo_database.add_galleryinfo(galleryinfo)
if not await self.info_database.get_info(index):
logger.debug(f"{index} couldn't find that info locally.")
await self.info_database.add_info(
Info.from_galleryinfo(galleryinfo)
)
logger.debug(f"Added info {index}.")
else:
logger.debug(f"{index} already has info locally.")
logger.debug(f"Added galleryinfo {index}.")
else:
logger.warning(f"{index} can't get galleryinfo from hitomi.la.")
continue
else:
logger.debug(f"{index} already has galleryinfo locally.")
# If there is galleryinfo, run it because there is also info
# galleryinfo가 있다면 info도 있기 때문에 실행
if not await self.info_database.get_info(index):
logger.debug(f"{index} couldn't find that info locally.")
if info := await self.request.get_info(index):
logger.debug(f"{index} can get info from hitomi.la.")
await self.info_database.add_infos([info])
logger.debug(f"Added info {index}.")
else:
logger.warning(f"{index} can't get info from hitomi.la.")
else:
logger.debug(f"{index} already has info locally.")
async def start(self, delay: float) -> NoReturn:
while True:
if index_list := await self.compare_index_list():

70
heliotrope/types.py
View file

@ -29,20 +29,20 @@ SetupTask = Task[NoReturn]
class _HitomiFileJSONOptional(TypedDict, total=False):
hasavifsmalltn: Literal[1]
hasavif: Literal[1]
class HitomiFileJSON(_HitomiFileJSONOptional):
width: int
hash: str
haswebp: Literal[0, 1]
hasavif: Literal[0, 1]
name: str
height: int
class _HitomiTagJSONOptional(TypedDict, total=False):
male: Optional[Literal["", "1"]]
female: Optional[Literal["", "1"]]
male: Literal["", "1", 1]
female: Literal["", "1", 1]
class HitomiTagJSON(_HitomiTagJSONOptional):
@ -50,22 +50,66 @@ class HitomiTagJSON(_HitomiTagJSONOptional):
tag: str
class HitomiGalleryinfoJSON(TypedDict):
date: str
title: str
type: str
japanese_title: Optional[str]
language: Optional[str]
files: list[HitomiFileJSON]
id: Union[str, int]
class HitomiParodysJSON(TypedDict):
parody: str
url: str
class HitomiArtistsJSON(TypedDict):
artist: str
url: str
class HitomiCharatersJSON(TypedDict):
character: str
url: str
class HitomiGroupsJSON(TypedDict):
group: str
url: str
class HitomiLanguagesJSON(TypedDict):
url: str
name: str
galleryid: str
language_localname: str
tags: list[HitomiTagJSON]
class HitomiGalleryinfoJSON(TypedDict):
# Union is for conversion.
id: Union[str, int]
# Literal["manga", "doujinshi", "gamecg", "aritstcg", "anime"]
type: str
# title
title: str
japanese_title: Optional[str]
# video
video: Optional[str]
videofilename: Optional[str]
# language
language_url: Optional[str]
language_localname: Optional[str]
language: Optional[str]
languages: list[HitomiLanguagesJSON]
# tags
artists: Optional[list[HitomiArtistsJSON]]
characters: Optional[list[HitomiCharatersJSON]]
parodys: Optional[list[HitomiParodysJSON]]
groups: Optional[list[HitomiGroupsJSON]]
files: list[HitomiFileJSON]
tags: Optional[list[HitomiTagJSON]]
# etc
scene_indexes: list[int]
related: list[int]
date: str
class HitomiInfoJSON(TypedDict):
id: Union[str, int]
title: str
thumbnail: str
thumbnail: HitomiFileJSON
artist: list[str]
group: list[str]
type: str

3
requirements/deps.txt
View file

@ -4,7 +4,8 @@ sentry-sdk==1.5.3
beautifulsoup4==4.10.0
types-beautifulsoup4==4.10.10
SQLAlchemy[mypy,asyncio]==1.4.31
ameilisearch==0.3.4
motor==2.5.1
dnspython==2.2.0
lxml==4.7.1
asyncpg==0.25.0
Js2Py==0.71

190
tests/common.py
View file

@ -1,291 +1,287 @@
galleryinfo = {
"id": 1613730,
"type": "manga",
"date": "2020-04-16 12:13:00-05",
"title": "Sekigahara-san wa Dasaretai | 세키가하라는 발산하고싶어",
"japanese_title": None,
"video": None,
"videofilename": None,
"language_url": None,
"language_localname": "한국어",
"language": "korean",
"date": "2020-04-16 12:13:00-05",
"languages": [],
"files": [
{
"width": 212,
"width": 1359,
"hash": "e8792b2ba4597a63d2ec69247f3bf5193ab33a8572699f47095c91695cbcb1d2",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809000.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "379070aaf2fe8a8adc59c44086847f85d46e6215a74142e1a09065783cc1aa6d",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809001.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "fda84c6d30ce1b5c3c779916bf8e906e07d64e6661456bc17ffceb5d78b749f4",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809002.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "72155754897a4ab7f9c54160cc63d0c95c048ec644656419510f910ec88eff3e",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809003.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "9ea9d7ce2941b25b44162b7911684b06ebbcad864a6d579f2f6b2c61e37b0dab",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809004.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "e8225de772608e84d8ba55e3c990ff3387118777717391c6a67048476c3d3d35",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809005.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "6df16bea287a5178feabfc62b94d64bb7613ca4eb5d1cc8140ec1509fe9b25ff",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809006.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "e63c239e150b41659aa09c4819ef0b6c1fc72c62f7c8cac2012bfd024d605aa4",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809007.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "38c5848c2b362a36fac1cc8db626d1b1ba0ba0d66c487b1751e82e01dfe831e6",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809008.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "901036980d701f6d5953cb85d0d6a692679f50e036d50d72e4eb047532bf46d1",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809009.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "61907339fb86534980285f12ae9c3a6f2d9eb4afd4303022be53a4d6e2c8ef6f",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809010.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "0bf8806c8c556d10be9baf3f01c087950844dcba2e16115272ec8ee0c5592ad3",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809011.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "9368589df1bb6a84ad293119ef90d5f170a60f8342a796344d17bbbd6790c5b2",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809012.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "a1acf266bacb2ec85b8284b608c23756a5073489330e9e764178b320a27d317e",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809013.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "38cafff51e479bacba2b79547a65b04546bc6bc28a9c0f56f4604ffa682c2375",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809014.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "57f08d603e66e6d7039432c66b009d83c795c5bf06b9bc4a7d65f7c964941b66",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809015.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "e9312d96f24bba371a1dcf6650ee3ddbf30932764f8ccd2fef61e321f7376951",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809016.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "3bc61f23e1ea94d52a8f9f6f2c7c074f66f8cdb5400f8d7cb024798caf2bdab4",
"haswebp": 1,
"hasavifsmalltn": 1,
"name": "809017.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "be2346c556c050cd2466d49b47e060e429e771f55ace51b8f7e5d4000aa91604",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809018.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "f70d8eab3446a95b92450b89a8bc2b3d451ed288fdacb158c05086f392bc3d7e",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809019.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
{
"width": 212,
"width": 1359,
"hash": "fc9cce2b732ed29d75b34844173df045eb055367fe2d366648e1b28aee4856a6",
"haswebp": 1,
"hasavifsmalltn": 0,
"name": "809020.png",
"height": 300,
"hasavif": 1,
"height": 1920,
},
],
"related": [1556534, 1379041, 1570712, 1379052, 1333507],
"scene_indexes": [],
"tags": [
{
"female": "1",
"male": "",
"url": "/tag/female%3Aahegao-all.html",
"tag": "ahegao",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Abig%20breasts-all.html",
"tag": "big breasts",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Adefloration-all.html",
"tag": "defloration",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Aleg%20lock-all.html",
"tag": "leg lock",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Alingerie-all.html",
"tag": "lingerie",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Anakadashi-all.html",
"tag": "nakadashi",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Apaizuri-all.html",
"tag": "paizuri",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Aponytail-all.html",
"tag": "ponytail",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Asole%20female-all.html",
"tag": "sole female",
"male": "",
"female": "1",
},
{
"female": "1",
"male": "",
"url": "/tag/female%3Asweating-all.html",
"tag": "sweating",
"male": "",
"female": "1",
},
{
"female": "",
"male": "1",
"url": "/tag/male%3Asole%20male-all.html",
"tag": "sole male",
"male": "1",
"female": "",
},
{
"female": "",
"male": "1",
"url": "/tag/male%3Asweating-all.html",
"tag": "sweating",
"male": "1",
"female": "",
},
{
"female": "",
"male": "1",
"url": "/tag/male%3Avirginity-all.html",
"tag": "virginity",
"male": "1",
"female": "",
},
],
"japanese_title": None,
"title": "Sekigahara-san wa Dasaretai | 세키가하라는 발산하고싶어",
"id": "1613730",
"type": "manga",
"artists": [{"artist": "tsukako", "url": "/artist/tsukako-all.html"}],
"characters": None,
"groups": None,
"parodys": None,
}
info = {
"id": "1613730",
"id": 1613730,
"title": "Sekigahara-san wa Dasaretai | 세키가하라는 발산하고싶어",
"thumbnail": "//tn.hitomi.la/smallbigtn/2/1d/e8792b2ba4597a63d2ec69247f3bf5193ab33a8572699f47095c91695cbcb1d2.jpg",
"thumbnail": {
"width": 1359,
"hash": "e8792b2ba4597a63d2ec69247f3bf5193ab33a8572699f47095c91695cbcb1d2",
"haswebp": 1,
"name": "809000.png",
"hasavif": 1,
"height": 1920,
},
"artist": ["tsukako"],
"group": [],
"type": "manga",
"language": "한국어",
"language": "korean",
"series": [],
"character": [],
"tags": [

3
tests/config.json
View file

@ -1,8 +1,7 @@
{
"INDEX_FILE": "index-korean.nozomi",
"MIRRORING_DELAY": 3600,
"INFO_DB_API_KEY": "masterKey",
"REFRESH_COMMON_JS_DELAY": 86400,
"INFO_DB_URL": "http://127.0.0.1:7700",
"INFO_DB_URL": "mongodb://root:test@127.0.0.1",
"GALLERYINFO_DB_URL": "postgresql+asyncpg://postgres:test@localhost/test_heliotrope"
}

11
tests/conftest.py
View file

@ -2,6 +2,7 @@ import json
from asyncio.events import AbstractEventLoop, get_running_loop, new_event_loop
from pytest import fixture, mark
from js2py.pyjs import undefined
from sanic_ext.extensions.http.extension import HTTPExtension
from sanic_ext.extensions.injection.extension import InjectionExtension
from sanic_ext.extensions.openapi.extension import OpenAPIExtension
@ -39,13 +40,13 @@ def get_config():
async def startup_test(heliotrope: Heliotrope, loop: AbstractEventLoop):
await heliotrope.ctx.orm.add_galleryinfo(Galleryinfo.from_dict(galleryinfo))
await heliotrope.ctx.meilisearch.add_infos([Info.from_dict(info)])
await heliotrope.ctx.odm.add_info(Info.from_dict(info))
async def closeup_test(heliotrope: Heliotrope, loop: AbstractEventLoop):
async with heliotrope.ctx.orm.engine.begin() as connection:
await connection.run_sync(mapper_registry.metadata.drop_all)
await heliotrope.ctx.meilisearch.index.delete()
await heliotrope.ctx.odm.collection.delete_many({})
@fixture
@ -77,10 +78,8 @@ def reset_extensions():
async def image_url():
hitomi_request = await HitomiRequest.setup()
common_js = await CommonJS.setup(hitomi_request)
yield await common_js.image_url_from_image(
galleryinfo["id"],
galleryinfo["files"][0],
True,
yield common_js.interpreter.url_from_url_from_hash(
galleryinfo["id"], galleryinfo["files"][0], "webp", undefined, "a"
)

2
tests/test_arg.py
View file

@ -34,7 +34,7 @@ def test_parse_args_with_config():
config.GALLERYINFO_DB_URL
== "postgresql+asyncpg://postgres:test@localhost/test_heliotrope"
)
assert config.INFO_DB_URL == "http://127.0.0.1:7700"
assert config.INFO_DB_URL == "mongodb://root:test@127.0.0.1"
assert config.INDEX_FILE == "index-korean.nozomi"
assert config.MIRRORING_DELAY == 3600
assert config.REFRESH_COMMON_JS_DELAY == 86400

5
tests/test_mirroring.py
View file

@ -14,7 +14,8 @@ async def test_mirroring_task(fake_app: Heliotrope, event_loop: AbstractEventLoo
try:
await wait_for(MirroringTask.setup(fake_app, 5), 15)
except TimeoutError:
info_total = await fake_app.ctx.meilisearch.get_total()
info_total = await fake_app.ctx.odm.get_all_index()
galleryinfo_total = await fake_app.ctx.orm.get_all_index()
assert len(galleryinfo_total) >= 1
assert info_total >= 1
assert len(info_total) >= 1