diff --git a/.gitignore b/.gitignore index 1437c53..9dfdf27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,34 +1,12 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.js - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# local env files -.env.local -.env.development.local -.env.test.local -.env.production.local - -# vercel -.vercel +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Postgresql stuffs +pg_data/ +pg_backups/ \ No newline at end of file diff --git a/README.md b/README.md index 3c581f8..bba8845 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,17 @@ -# PZ Server -The Photo-z (PZ) Server is one of the Brazilian in-kind contributions offered by the Laboratório Interinstitucional de e-Astronomia (LIneA) to the PZ Coordination Group from Vera C. Rubin Observatory's Data Management (DM) Team. Inspired by features of the DES Science Portal (Gschwend et al., 2018; Fausti Neto et al., 2018), the PZ Server is being planned to be an online service, complementary to the Rubin Science Platform (RSP), to host PZ-related lightweight data products and to offer data management tools that allow sharing data products among RSP users, attach and share relevant metadata, and help on provenance tracking. ---- +### -## Development -### Dependencies - - [NodeJS](https://nodejs.org/en/download/) - - [Yarn](https://classic.yarnpkg.com/lang/en/docs/install/#debian-stable) +### Setup Database -First, install the dependencies: -```bash -yarn install -``` +Inicie o serviço do banco de dados e espere a mensagem `database system is ready to accept connections` para só depois iniciar o backend. -After, run the development server: ```bash -yarn dev -``` - -Finally, open [http://localhost:3000](http://localhost:3000) with your browser. - ---- - -## Production -### Dependencies - - [Docker](https://docs.docker.com/engine/install/) - - [Docker Compose](https://docs.docker.com/compose/install/) - -Build the application's image: -```bash -docker-compose build -``` - -In case you want to use an already built image, change the name in [docker-compose.yml](docker-compose.yml): -``` -... -services: - frontend: - image: CHANGE_IMAGE_NAME - ... +docker-compose up database ``` -Run the application: +### Setup Frontend +Antes de iniciar o container do frontend é necessário instalar as dependencias do react. ```bash -docker-compose up +docker-compose run frontend yarn ``` diff --git a/archive/.gitignore b/archive/.gitignore new file mode 100644 index 0000000..f92e47d --- /dev/null +++ b/archive/.gitignore @@ -0,0 +1,4 @@ +* +*/ +!.gitignore +!django_static diff --git a/archive/django_static/.gitignore b/archive/django_static/.gitignore new file mode 100644 index 0000000..e11f08e --- /dev/null +++ b/archive/django_static/.gitignore @@ -0,0 +1,4 @@ +* +*/ +!.gitignore + diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..fb56c4d --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.10-slim as builder + +WORKDIR /app + +# If this is set to a non-empty string, Python won’t try +# to write .pyc files on the import of source modules +ENV PYTHONDONTWRITEBYTECODE=1 + +# Force the stdout and stderr streams to be unbuffered. +# This option has no effect on the stdin stream. +ENV PYTHONUNBUFFERED=1 + +# TODO: Instalar as dependencias do LDAP e do Shibboleth. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc \ + python3-dev \ + build-essential \ + libpcre3 \ + libpcre3-dev + +RUN python -m venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +COPY requirements.txt . + +RUN pip install --upgrade pip wheel && \ + pip install -r requirements.txt + + +FROM python:3.10-slim + +COPY --from=builder /opt/venv /opt/venv + +WORKDIR /app + +ENV PATH="/opt/venv/bin:$PATH" + +COPY ./entrypoint.sh /app/entrypoint.sh +ENTRYPOINT ./entrypoint.sh + +COPY . /app + diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..7b5ea11 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,17 @@ +# Pz Server API +First version of Pz Server data access API for LSST + +## Requirements +- docker +- docker-compose + +## Getting Started + +```bash +docker-compose up -d +``` + +Open http://localhost with your browser to see the result. + +## License +[MIT](LICENSE) diff --git a/backend/app/LICENSE b/backend/app/LICENSE new file mode 100644 index 0000000..90f1032 --- /dev/null +++ b/backend/app/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 LIneA IT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/admin.py b/backend/app/core/admin.py new file mode 100644 index 0000000..faf4cc0 --- /dev/null +++ b/backend/app/core/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from core.models import ProductType, Product, Release + +admin.site.register(ProductType) +admin.site.register(Release) +admin.site.register(Product) diff --git a/backend/app/core/api/serializers.py b/backend/app/core/api/serializers.py new file mode 100644 index 0000000..ebd0839 --- /dev/null +++ b/backend/app/core/api/serializers.py @@ -0,0 +1,21 @@ +from rest_framework import serializers +from core import models + + +class ProductTypeSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProductType + fields = "__all__" + + +class ReleaseSerializer(serializers.ModelSerializer): + class Meta: + model = models.Release + fields = "__all__" + + +class ProductSerializer(serializers.ModelSerializer): + class Meta: + model = models.Product + fields = "__all__" + read_only_fields = ("file_size", "file_extension", "file_name",) diff --git a/backend/app/core/api/viewsets.py b/backend/app/core/api/viewsets.py new file mode 100644 index 0000000..d3d56e9 --- /dev/null +++ b/backend/app/core/api/viewsets.py @@ -0,0 +1,29 @@ +from rest_framework import viewsets +from core.api import serializers +from core import models +import os + + +class ProductTypeViewSet(viewsets.ModelViewSet): + queryset = models.ProductType.objects.all() + serializer_class = serializers.ProductTypeSerializer + + +class ReleaseViewSet(viewsets.ModelViewSet): + queryset = models.Release.objects.all() + serializer_class = serializers.ReleaseSerializer + + +class ProductViewSet(viewsets.ModelViewSet): + queryset = models.Product.objects.all() + serializer_class = serializers.ProductSerializer + + def perform_create(self, serializer): + main_file = self.request.data.get("main_file") + file_size = main_file.size + file_name, file_extension = os.path.splitext(main_file.name) + + serializer.save( + file_size=file_size, file_name=file_name, + file_extension=file_extension + ) diff --git a/backend/app/core/apps.py b/backend/app/core/apps.py new file mode 100644 index 0000000..8115ae6 --- /dev/null +++ b/backend/app/core/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'core' diff --git a/backend/app/core/migrations/0001_initial.py b/backend/app/core/migrations/0001_initial.py new file mode 100644 index 0000000..5b3dcdf --- /dev/null +++ b/backend/app/core/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# Generated by Django 4.0.2 on 2022-02-22 22:55 + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ProductType', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('display_name', models.CharField(max_length=255)), + ('attributes', models.JSONField(default=dict)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Product', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('display_name', models.CharField(max_length=255)), + ('product_dir', models.CharField(max_length=255)), + ('file_name', models.CharField(max_length=120)), + ('main_file', models.FileField(upload_to=core.models.upload_file_products)), + ('file_size', models.IntegerField()), + ('file_extension', models.CharField(max_length=10)), + ('official_product', models.BooleanField(default=False)), + ('survey', models.CharField(max_length=255)), + ('pz_code', models.CharField(max_length=55)), + ('description', models.TextField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('product_type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='products', to='core.producttype')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='products', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/backend/app/core/migrations/0002_release_product_description_file_and_more.py b/backend/app/core/migrations/0002_release_product_description_file_and_more.py new file mode 100644 index 0000000..bbb21e6 --- /dev/null +++ b/backend/app/core/migrations/0002_release_product_description_file_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.0.2 on 2022-02-24 20:10 + +import core.models +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Release', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('display_name', models.CharField(max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.AddField( + model_name='product', + name='description_file', + field=models.FileField(blank=True, null=True, upload_to=core.models.upload_file_products), + ), + migrations.AlterField( + model_name='product', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='product', + name='pz_code', + field=models.CharField(blank=True, max_length=55, null=True), + ), + migrations.AlterField( + model_name='product', + name='survey', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='producttype', + name='attributes', + field=models.JSONField(blank=True, default=dict, null=True), + ), + migrations.AddField( + model_name='product', + name='release', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='products', to='core.release'), + ), + ] diff --git a/backend/app/core/migrations/0003_remove_product_product_dir.py b/backend/app/core/migrations/0003_remove_product_product_dir.py new file mode 100644 index 0000000..62e6eda --- /dev/null +++ b/backend/app/core/migrations/0003_remove_product_product_dir.py @@ -0,0 +1,17 @@ +# Generated by Django 4.0.2 on 2022-03-07 17:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_release_product_description_file_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='product', + name='product_dir', + ), + ] diff --git a/backend/app/core/migrations/__init__.py b/backend/app/core/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/models.py b/backend/app/core/models.py new file mode 100644 index 0000000..7657bf2 --- /dev/null +++ b/backend/app/core/models.py @@ -0,0 +1,60 @@ +from django.db import models +from django.contrib.auth.models import User + + +class ProductType(models.Model): + name = models.CharField(max_length=255) + display_name = models.CharField(max_length=255) + attributes = models.JSONField(default=dict, null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.display_name} ({self.name})" + + +class Release(models.Model): + name = models.CharField(max_length=255) + display_name = models.CharField(max_length=255) + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.display_name} ({self.name})" + + +def upload_file_products(instance, filename): + return f"{filename}" + + +class Product(models.Model): + display_name = models.CharField(max_length=255) + # product_dir = models.CharField(max_length=255) + main_file = models.FileField( + upload_to=upload_file_products, null=False, blank=False + ) + file_name = models.CharField(max_length=120) + file_size = models.IntegerField() + file_extension = models.CharField(max_length=10) + description_file = models.FileField( + upload_to=upload_file_products, null=True, blank=True + ) + official_product = models.BooleanField(default=False) + survey = models.CharField(max_length=255, null=True, blank=True) + pz_code = models.CharField(max_length=55, null=True, blank=True) + description = models.TextField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + user = models.ForeignKey( + User, on_delete=models.PROTECT, related_name="products" + ) + product_type = models.ForeignKey( + ProductType, on_delete=models.PROTECT, related_name="products" + ) + release = models.ForeignKey( + Release, + on_delete=models.PROTECT, + related_name="products", + null=True, + blank=True, + ) + + def __str__(self): + return f"{self.display_name} ({self.file_name})" diff --git a/backend/app/core/tests.py b/backend/app/core/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/backend/app/core/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/app/core/views.py b/backend/app/core/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/backend/app/core/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/backend/app/manage.py b/backend/app/manage.py new file mode 100755 index 0000000..7ec1220 --- /dev/null +++ b/backend/app/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/backend/app/pzserver/__init__.py b/backend/app/pzserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/pzserver/asgi.py b/backend/app/pzserver/asgi.py new file mode 100644 index 0000000..e6a72a6 --- /dev/null +++ b/backend/app/pzserver/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for pzserver project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + +application = get_asgi_application() diff --git a/backend/app/pzserver/settings.py b/backend/app/pzserver/settings.py new file mode 100644 index 0000000..27a16a9 --- /dev/null +++ b/backend/app/pzserver/settings.py @@ -0,0 +1,151 @@ +""" +Django settings for pzserver project. + +Generated by 'django-admin startproject' using Django 4.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +# BASE_DIR = Path(__file__).resolve().parent.parent +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv("SECRET_KEY") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = int(os.getenv("DEBUG", 1)) + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + # Third-party + "rest_framework", + "corsheaders", + # Apps + "core", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "pzserver.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "pzserver.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.0/ref/settings/#databases + + +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql', +# 'NAME': os.environ.get('POSTGRES_NAME'), +# 'USER': os.environ.get('POSTGRES_USER'), +# 'PASSWORD': os.environ.get('POSTGRES_PASSWORD'), +# 'HOST': 'db', +# 'PORT': 5432, +# } +# } + +DATABASES = { + "default": { + "ENGINE": os.getenv("SQL_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.getenv( + "SQL_DATABASE", os.path.join(BASE_DIR, "db.sqlite3") + ), + "USER": os.getenv("SQL_USER", "user"), + "PASSWORD": os.getenv("SQL_PASSWORD", "password"), + "HOST": os.getenv("SQL_HOST", "localhost"), + "PORT": os.getenv("SQL_PORT", "5432"), + } +} + +# Password validation +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators + +auth_pass_str = "django.contrib.auth.password_validation" + +AUTH_PASSWORD_VALIDATORS = [ + {"NAME": f"{auth_pass_str}.UserAttributeSimilarityValidator"}, + {"NAME": f"{auth_pass_str}.MinimumLengthValidator"}, + {"NAME": f"{auth_pass_str}.CommonPasswordValidator"}, + {"NAME": f"{auth_pass_str}.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.0/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = "/static/" +STATIC_ROOT = "/static/" + +MEDIA_URL = "/archive/data/" +MEDIA_ROOT = "/archive/data/" + + +# Default primary key field type +# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +# CORS + +# 'DJANGO_ALLOWED_HOSTS' should be a single string of hosts with a space +# between each. For example: 'DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]' +# CORS_ALLOWED_ORIGINS = +ALLOWED_HOSTS = os.getenv( + "DJANGO_ALLOWED_HOSTS", '*' +).split(" ") diff --git a/backend/app/pzserver/urls.py b/backend/app/pzserver/urls.py new file mode 100644 index 0000000..4c5d9cf --- /dev/null +++ b/backend/app/pzserver/urls.py @@ -0,0 +1,50 @@ +"""pzserver URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include +# from django import settings + +from rest_framework import routers +from core.api import viewsets as products_viewsets + +route = routers.DefaultRouter() + +route.register( + r"product-types", + products_viewsets.ProductTypeViewSet, basename="ProductTypes" +) +route.register( + r"products", + products_viewsets.ProductViewSet, basename="Products" +) +route.register( + r"releases", + products_viewsets.ReleaseViewSet, basename="Releases" +) + +urlpatterns = [ + path("admin/", admin.site.urls), + path("", include(route.urls)), +] + +# if settings.DEBUG: +# from django.views.static import serve +# from django.conf.urls.static import static + +# urlpatterns += [ +# url(r'^media/(?P.*)$', serve, +# {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}), +# ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/backend/app/pzserver/wsgi.py b/backend/app/pzserver/wsgi.py new file mode 100644 index 0000000..94eea3a --- /dev/null +++ b/backend/app/pzserver/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for pzserver project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + +application = get_wsgi_application() diff --git a/backend/core/__init__.py b/backend/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/core/admin.py b/backend/core/admin.py new file mode 100644 index 0000000..faf4cc0 --- /dev/null +++ b/backend/core/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from core.models import ProductType, Product, Release + +admin.site.register(ProductType) +admin.site.register(Release) +admin.site.register(Product) diff --git a/backend/core/api/serializers.py b/backend/core/api/serializers.py new file mode 100644 index 0000000..ebd0839 --- /dev/null +++ b/backend/core/api/serializers.py @@ -0,0 +1,21 @@ +from rest_framework import serializers +from core import models + + +class ProductTypeSerializer(serializers.ModelSerializer): + class Meta: + model = models.ProductType + fields = "__all__" + + +class ReleaseSerializer(serializers.ModelSerializer): + class Meta: + model = models.Release + fields = "__all__" + + +class ProductSerializer(serializers.ModelSerializer): + class Meta: + model = models.Product + fields = "__all__" + read_only_fields = ("file_size", "file_extension", "file_name",) diff --git a/backend/core/api/viewsets.py b/backend/core/api/viewsets.py new file mode 100644 index 0000000..d3d56e9 --- /dev/null +++ b/backend/core/api/viewsets.py @@ -0,0 +1,29 @@ +from rest_framework import viewsets +from core.api import serializers +from core import models +import os + + +class ProductTypeViewSet(viewsets.ModelViewSet): + queryset = models.ProductType.objects.all() + serializer_class = serializers.ProductTypeSerializer + + +class ReleaseViewSet(viewsets.ModelViewSet): + queryset = models.Release.objects.all() + serializer_class = serializers.ReleaseSerializer + + +class ProductViewSet(viewsets.ModelViewSet): + queryset = models.Product.objects.all() + serializer_class = serializers.ProductSerializer + + def perform_create(self, serializer): + main_file = self.request.data.get("main_file") + file_size = main_file.size + file_name, file_extension = os.path.splitext(main_file.name) + + serializer.save( + file_size=file_size, file_name=file_name, + file_extension=file_extension + ) diff --git a/backend/core/apps.py b/backend/core/apps.py new file mode 100644 index 0000000..8115ae6 --- /dev/null +++ b/backend/core/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'core' diff --git a/backend/core/migrations/0001_initial.py b/backend/core/migrations/0001_initial.py new file mode 100644 index 0000000..5b3dcdf --- /dev/null +++ b/backend/core/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# Generated by Django 4.0.2 on 2022-02-22 22:55 + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ProductType', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('display_name', models.CharField(max_length=255)), + ('attributes', models.JSONField(default=dict)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Product', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('display_name', models.CharField(max_length=255)), + ('product_dir', models.CharField(max_length=255)), + ('file_name', models.CharField(max_length=120)), + ('main_file', models.FileField(upload_to=core.models.upload_file_products)), + ('file_size', models.IntegerField()), + ('file_extension', models.CharField(max_length=10)), + ('official_product', models.BooleanField(default=False)), + ('survey', models.CharField(max_length=255)), + ('pz_code', models.CharField(max_length=55)), + ('description', models.TextField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('product_type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='products', to='core.producttype')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='products', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/backend/core/migrations/0002_release_product_description_file_and_more.py b/backend/core/migrations/0002_release_product_description_file_and_more.py new file mode 100644 index 0000000..bbb21e6 --- /dev/null +++ b/backend/core/migrations/0002_release_product_description_file_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.0.2 on 2022-02-24 20:10 + +import core.models +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Release', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('display_name', models.CharField(max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.AddField( + model_name='product', + name='description_file', + field=models.FileField(blank=True, null=True, upload_to=core.models.upload_file_products), + ), + migrations.AlterField( + model_name='product', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='product', + name='pz_code', + field=models.CharField(blank=True, max_length=55, null=True), + ), + migrations.AlterField( + model_name='product', + name='survey', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='producttype', + name='attributes', + field=models.JSONField(blank=True, default=dict, null=True), + ), + migrations.AddField( + model_name='product', + name='release', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='products', to='core.release'), + ), + ] diff --git a/backend/core/migrations/0003_remove_product_product_dir.py b/backend/core/migrations/0003_remove_product_product_dir.py new file mode 100644 index 0000000..62e6eda --- /dev/null +++ b/backend/core/migrations/0003_remove_product_product_dir.py @@ -0,0 +1,17 @@ +# Generated by Django 4.0.2 on 2022-03-07 17:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_release_product_description_file_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='product', + name='product_dir', + ), + ] diff --git a/backend/core/migrations/__init__.py b/backend/core/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/core/models.py b/backend/core/models.py new file mode 100644 index 0000000..7657bf2 --- /dev/null +++ b/backend/core/models.py @@ -0,0 +1,60 @@ +from django.db import models +from django.contrib.auth.models import User + + +class ProductType(models.Model): + name = models.CharField(max_length=255) + display_name = models.CharField(max_length=255) + attributes = models.JSONField(default=dict, null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.display_name} ({self.name})" + + +class Release(models.Model): + name = models.CharField(max_length=255) + display_name = models.CharField(max_length=255) + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.display_name} ({self.name})" + + +def upload_file_products(instance, filename): + return f"{filename}" + + +class Product(models.Model): + display_name = models.CharField(max_length=255) + # product_dir = models.CharField(max_length=255) + main_file = models.FileField( + upload_to=upload_file_products, null=False, blank=False + ) + file_name = models.CharField(max_length=120) + file_size = models.IntegerField() + file_extension = models.CharField(max_length=10) + description_file = models.FileField( + upload_to=upload_file_products, null=True, blank=True + ) + official_product = models.BooleanField(default=False) + survey = models.CharField(max_length=255, null=True, blank=True) + pz_code = models.CharField(max_length=55, null=True, blank=True) + description = models.TextField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + user = models.ForeignKey( + User, on_delete=models.PROTECT, related_name="products" + ) + product_type = models.ForeignKey( + ProductType, on_delete=models.PROTECT, related_name="products" + ) + release = models.ForeignKey( + Release, + on_delete=models.PROTECT, + related_name="products", + null=True, + blank=True, + ) + + def __str__(self): + return f"{self.display_name} ({self.file_name})" diff --git a/backend/core/tests.py b/backend/core/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/backend/core/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/core/views.py b/backend/core/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/backend/core/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh new file mode 100755 index 0000000..e9a20af --- /dev/null +++ b/backend/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +python manage.py migrate --noinput +python manage.py collectstatic --noinput + +gunicorn pzserver.wsgi:application --bind 0.0.0.0:8000 --reload + +# TODO: Para produção é necessário usar o uWSGI! +# uWSGI para servir o app e ter compatibilidade com Shibboleth +# https://uwsgi-docs.readthedocs.io/en/latest/WSGIquickstart.html +# TODO: Em produção não é recomendado o auto reload. utilizar uma variavel de ambiente para ligar ou desligar esta opção. +# uwsgi \ +# --socket 0.0.0.0:8000 \ +# # --wsgi-file /app/pzserver/wsgi.py \ +# # --static-map /django_static/admin=/app/static/admin \ +# # --static-map /django_static/rest_framework=/app/static/rest_framework \ +# --module pzserver.wsgi:application \ +# --buffer-size=32768 \ +# --processes=4 \ +# --threads=2 \ +# --py-autoreload=1 + + + diff --git a/backend/manage.py b/backend/manage.py new file mode 100755 index 0000000..7ec1220 --- /dev/null +++ b/backend/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/backend/pzserver/__init__.py b/backend/pzserver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/pzserver/asgi.py b/backend/pzserver/asgi.py new file mode 100644 index 0000000..e6a72a6 --- /dev/null +++ b/backend/pzserver/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for pzserver project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + +application = get_asgi_application() diff --git a/backend/pzserver/settings.py b/backend/pzserver/settings.py new file mode 100644 index 0000000..dbf99c0 --- /dev/null +++ b/backend/pzserver/settings.py @@ -0,0 +1,135 @@ +""" +Django settings for pzserver project. + +Generated by 'django-admin startproject' using Django 4.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +# BASE_DIR = Path(__file__).resolve().parent.parent +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv("SECRET_KEY") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = int(os.getenv("DEBUG", 1)) + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + # Third-party + "rest_framework", + "corsheaders", + # Apps + "core", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "pzserver.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "pzserver.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.0/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": os.getenv("SQL_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.getenv("SQL_DATABASE", os.path.join(BASE_DIR, "db.sqlite3")), + "USER": os.getenv("SQL_USER", "user"), + "PASSWORD": os.getenv("SQL_PASSWORD", "password"), + "HOST": os.getenv("SQL_HOST", "localhost"), + "PORT": os.getenv("SQL_PORT", "5432"), + } +} + +# Password validation +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators + +auth_pass_str = "django.contrib.auth.password_validation" + +AUTH_PASSWORD_VALIDATORS = [ + {"NAME": f"{auth_pass_str}.UserAttributeSimilarityValidator"}, + {"NAME": f"{auth_pass_str}.MinimumLengthValidator"}, + {"NAME": f"{auth_pass_str}.CommonPasswordValidator"}, + {"NAME": f"{auth_pass_str}.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.0/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = "/django_static/" +STATIC_ROOT = "/archive/django_static/" + +MEDIA_URL = "/archive/data/" +MEDIA_ROOT = "/archive/data/" + + +# Default primary key field type +# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +# CORS + +# 'DJANGO_ALLOWED_HOSTS' should be a single string of hosts with a space +# between each. For example: 'DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]' +# CORS_ALLOWED_ORIGINS = +ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "*").split(" ") diff --git a/backend/pzserver/urls.py b/backend/pzserver/urls.py new file mode 100644 index 0000000..42360aa --- /dev/null +++ b/backend/pzserver/urls.py @@ -0,0 +1,44 @@ +"""pzserver URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include + +# from django import settings + +from rest_framework import routers +from core.api import viewsets as products_viewsets + +route = routers.DefaultRouter() + +route.register( + r"product-types", products_viewsets.ProductTypeViewSet, basename="ProductTypes" +) +route.register(r"products", products_viewsets.ProductViewSet, basename="Products") +route.register(r"releases", products_viewsets.ReleaseViewSet, basename="Releases") + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", include(route.urls)), +] + +# if settings.DEBUG: +# from django.views.static import serve +# from django.conf.urls.static import static + +# urlpatterns += [ +# url(r'^media/(?P.*)$', serve, +# {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}), +# ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/backend/pzserver/wsgi.py b/backend/pzserver/wsgi.py new file mode 100644 index 0000000..94eea3a --- /dev/null +++ b/backend/pzserver/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for pzserver project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pzserver.settings') + +application = get_wsgi_application() diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..ecf905e --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,7 @@ +Django==4.0.2 +djangorestframework==3.13.1 +psycopg2-binary==2.9.3 +django-cors-headers==3.11.0 +django-filebrowser-no-grappelli==4.0.1 +uWSGI==2.0.20 +gunicorn==20.1.0 \ No newline at end of file diff --git a/demo.ipynb b/demo.ipynb deleted file mode 100644 index ccd7d58..0000000 --- a/demo.ipynb +++ /dev/null @@ -1,474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - " \n", - " \n", - "\n", - "## Photo-z Server - Demo Notebook\n", - "**Contact**: [Julia Gschwend](mailto:julia@linea.gov.br)
\n", - "**Last verified run**: yyyy-mm-dd
\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "The Photo-z Server is an online service available for the LSST Community to host and share lightweight photo-z related data products. The upload and download of data and metadata is done at the website www.linea.gov.br/pz-server (RSP authentication required). The registered data products can be accessed directly from Python code using the Photo-z Server's data access API, as demonstrated below.
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pz_server import PzServer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server = PzServer()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get basic info from PZ Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_product_types" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_users" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_releases" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Spec-z Catalog \n", - "#### List Spec-z Catalogs available on Pz Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_products_available(user=\"PZ Coord. Group\", product_type=\"specz_catalog\", release=\"all\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get metadata of a given Spec-z Catalog " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_specz_1 = PzServer.get_product_metadata(product_id=\"0001\")\n", - "metadata_specz_1 # markdown table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_specz_2 = PzServer.get_product_metadata(product_id=\"0002\")\n", - "metadata_specz_2 # markdown table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_specz_3 = PzServer.get_product_metadata(product_id=\"0003\")\n", - "metadata_specz_3 # markdown table" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Retrieve a given Spec-z Catalog as Pandas DataFrame object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_specz_1 = PzServer.get_product(product_id=\"0001\", save_file=\"False\")\n", - "dataframe_specz_2 = PzServer.get_product(product_id=\"0002\", save_file=\"False\")\n", - "dataframe_specz_3 = PzServer.get_product(product_id=\"0003\", save_file=\"False\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Display basic statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_specz_1.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_specz_2.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_specz_3.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Mini QA Spec-z Catalogs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.qa_specz(data=[dataframe_specz_1, dataframe_specz_2, dataframe_specz_3],\n", - " metadata=[metadata_specz_1, metadata_specz_2, metadata_specz_3])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training and Validation Sets \n", - "#### List Training and Validation Sets available on Pz Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_products_available(user=\"PZ Coord. Group\", product_type=\"train_set\", release=\"all\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_products_available(user=\"PZ Coord. Group\", product_type=\"valid_set\", release=\"all\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get metadata of a given Training Set" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_train_set = PzServer.get_product_metadata(product_id=\"0004\")\n", - "metadata_train_set # markdown table" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get metadata of a given Validation Set" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_valid_set = PzServer.get_product_metadata(product_id=\"0005\")\n", - "metadata_valid_set # markdown table" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Retrieve a given Training (or Validation) Set as Pandas DataFrame object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_train_set = PzServer.get_product(product_id=\"0004\", save_file=\"False\")\n", - "dataframe_valid_set = PzServer.get_product(product_id=\"0005\", save_file=\"False\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Display basic statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_train_set.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_valid_set.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Mini QA Traininig and Validaiton Sets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.qa_train_valid(data=[dataframe_train_set, dataframe_valid_set], \n", - " metadata=[metadata_train_set, metadata_valid_set])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Photo-z Validation Results\n", - "#### List Photo-z Validation Results available on Pz Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_server.list_products_available(user=\"PZ Coord. Group\", product_type=\"pz_result\", release=\"all\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get metadata of a given Training Set" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata_pz_result = PzServer.get_product_metadata(product_id=\"0006\")\n", - "metadata_pz_result # markdown table" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Retrieve a given Photo-z Validation Result: download .tar file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pz_result = PzServer.get_product(product_id=\"0006\", save_file=\"True\")\n", - "pz_result # string tar file name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Basic Photo-z Validation Plots\n", - "\n", - "Reuse RAIL evaluation plots and tables for illustration with an example of Photo-z Validation Results. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# open .tar file " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from RAIL.examples.evaluation.utils import *\n", - "from RAIL.rail.evaluation.metrics.pit import *\n", - "from RAIL.rail.evaluation.metrics.cdeloss import *" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "my_path = 'xxx/xxx/xx' \n", - "pdfs_file = os.path.join(my_path, \"pdfs_FZBoost.hdf5\")\n", - "ztrue_file = os.path.join(my_path, \"ztrue_validation_set.hdf5\")\n", - "pdfs, zgrid, ztrue, photoz_mode = read_pz_output(pdfs_file, ztrue_file) # all numpy arrays" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Plot PIT-QQ" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plot_pit_qq(pdfs, zgrid, ztrue, title=\"PIT-QQ - toy data\", code=\"FZBoost\",\n", - " pit_out_rate=pit_out_rate, savefig=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "PZ Validation metrics table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "summary = Summary(pdfs, zgrid, ztrue)\n", - "summary.markdown_metrics_table(pitobj=pitobj) # pitobj as optional input to speed-up metrics evaluation" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.6" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docker-compose.yml b/docker-compose.yml index 86ece38..7dc04e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,47 @@ -version: "3.8" +version: "3.9" services: + database: + image: postgres:13.6-alpine + env_file: + - .env + expose: + # Deixa a porta do banco de dados acessivel + # para ferramentas externas como o dbeaver por exemplo. + - 5432 + volumes: + - ./pg_data:/var/lib/postgresql/data + - ./pg_backups:/pg_backups + + backend: + build: ./backend + env_file: + - .env + volumes: + - ./backend:/app + - ./archive/data:/archive/data + - ./archive/django_static:/archive/django_static + depends_on: + - database + frontend: - build: . + image: node:lts + working_dir: /app + volumes: + - ./frontend:/app + stdin_open: true + command: yarn dev + # ports: + # - 3000:3000 + + nginx: + image: nginx:1.21.6-alpine ports: - - 3000:3000 + - 80:8080 + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./archive/django_static:/var/www/django_static + - ./archive/data:/var/www/data + depends_on: + - backend + - frontend diff --git a/env_template b/env_template new file mode 100644 index 0000000..1fd8aeb --- /dev/null +++ b/env_template @@ -0,0 +1,20 @@ +# Postgres +POSTGRES_USER=pzadmin +POSTGRES_PASSWORD=adminpz +POSTGRES_DB=pzdev + +# Backend/Django +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG=1 +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY='#(y8c83h06$6+%*g*a@xm#4yw%hn@dx_!^^$+pey#()a7&' + +# Database +SQL_ENGINE=django.db.backends.postgresql +SQL_USER=pzadmin +SQL_PASSWORD=adminpz +SQL_DATABASE=pzdev +SQL_HOST=database +SQL_PORT=5432 + +# DJANGO_ALLOWED_HOSTS \ No newline at end of file diff --git a/.editorconfig b/frontend/.editorconfig similarity index 100% rename from .editorconfig rename to frontend/.editorconfig diff --git a/.eslintrc.json b/frontend/.eslintrc.json similarity index 100% rename from .eslintrc.json rename to frontend/.eslintrc.json diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..1437c53 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,34 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env.local +.env.development.local +.env.test.local +.env.production.local + +# vercel +.vercel diff --git a/Dockerfile b/frontend/Dockerfile similarity index 100% rename from Dockerfile rename to frontend/Dockerfile diff --git a/LICENSE b/frontend/LICENSE similarity index 100% rename from LICENSE rename to frontend/LICENSE diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..3c581f8 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,48 @@ +# PZ Server + +The Photo-z (PZ) Server is one of the Brazilian in-kind contributions offered by the Laboratório Interinstitucional de e-Astronomia (LIneA) to the PZ Coordination Group from Vera C. Rubin Observatory's Data Management (DM) Team. Inspired by features of the DES Science Portal (Gschwend et al., 2018; Fausti Neto et al., 2018), the PZ Server is being planned to be an online service, complementary to the Rubin Science Platform (RSP), to host PZ-related lightweight data products and to offer data management tools that allow sharing data products among RSP users, attach and share relevant metadata, and help on provenance tracking. + +--- + +## Development +### Dependencies + - [NodeJS](https://nodejs.org/en/download/) + - [Yarn](https://classic.yarnpkg.com/lang/en/docs/install/#debian-stable) + +First, install the dependencies: +```bash +yarn install +``` + +After, run the development server: +```bash +yarn dev +``` + +Finally, open [http://localhost:3000](http://localhost:3000) with your browser. + +--- + +## Production +### Dependencies + - [Docker](https://docs.docker.com/engine/install/) + - [Docker Compose](https://docs.docker.com/compose/install/) + +Build the application's image: +```bash +docker-compose build +``` + +In case you want to use an already built image, change the name in [docker-compose.yml](docker-compose.yml): +``` +... +services: + frontend: + image: CHANGE_IMAGE_NAME + ... +``` + +Run the application: +```bash +docker-compose up +``` diff --git a/components/Footer.js b/frontend/components/Footer.js similarity index 100% rename from components/Footer.js rename to frontend/components/Footer.js diff --git a/components/Header.js b/frontend/components/Header.js similarity index 99% rename from components/Header.js rename to frontend/components/Header.js index 004bc04..10fb095 100644 --- a/components/Header.js +++ b/frontend/components/Header.js @@ -33,7 +33,7 @@ function Header() { { description: 'Contact', href: '/contact' - } + }, ] return ( diff --git a/components/Link.js b/frontend/components/Link.js similarity index 100% rename from components/Link.js rename to frontend/components/Link.js diff --git a/contexts/AuthContext.js b/frontend/contexts/AuthContext.js similarity index 100% rename from contexts/AuthContext.js rename to frontend/contexts/AuthContext.js diff --git a/frontend/docker-compose.yml b/frontend/docker-compose.yml new file mode 100644 index 0000000..86ece38 --- /dev/null +++ b/frontend/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.8" + +services: + frontend: + build: . + ports: + - 3000:3000 diff --git a/next.config.js b/frontend/next.config.js similarity index 100% rename from next.config.js rename to frontend/next.config.js diff --git a/package.json b/frontend/package.json similarity index 100% rename from package.json rename to frontend/package.json diff --git a/pages/_app.js b/frontend/pages/_app.js similarity index 100% rename from pages/_app.js rename to frontend/pages/_app.js diff --git a/pages/_document.js b/frontend/pages/_document.js similarity index 100% rename from pages/_document.js rename to frontend/pages/_document.js diff --git a/pages/about.js b/frontend/pages/about.js similarity index 100% rename from pages/about.js rename to frontend/pages/about.js diff --git a/pages/contact.js b/frontend/pages/contact.js similarity index 100% rename from pages/contact.js rename to frontend/pages/contact.js diff --git a/pages/index.js b/frontend/pages/index.js similarity index 100% rename from pages/index.js rename to frontend/pages/index.js diff --git a/pages/login.js b/frontend/pages/login.js similarity index 100% rename from pages/login.js rename to frontend/pages/login.js diff --git a/pages/tutorials.js b/frontend/pages/tutorials.js similarity index 100% rename from pages/tutorials.js rename to frontend/pages/tutorials.js diff --git a/pages/upload.js b/frontend/pages/upload.js similarity index 100% rename from pages/upload.js rename to frontend/pages/upload.js diff --git a/prettier.config.js b/frontend/prettier.config.js similarity index 100% rename from prettier.config.js rename to frontend/prettier.config.js diff --git a/public/favicon.png b/frontend/public/favicon.png similarity index 100% rename from public/favicon.png rename to frontend/public/favicon.png diff --git a/public/header.jpg b/frontend/public/header.jpg similarity index 100% rename from public/header.jpg rename to frontend/public/header.jpg diff --git a/public/interfaces/lsst-dp.jpg b/frontend/public/interfaces/lsst-dp.jpg similarity index 100% rename from public/interfaces/lsst-dp.jpg rename to frontend/public/interfaces/lsst-dp.jpg diff --git a/public/interfaces/upload.jpg b/frontend/public/interfaces/upload.jpg similarity index 100% rename from public/interfaces/upload.jpg rename to frontend/public/interfaces/upload.jpg diff --git a/public/interfaces/user-dp.jpg b/frontend/public/interfaces/user-dp.jpg similarity index 100% rename from public/interfaces/user-dp.jpg rename to frontend/public/interfaces/user-dp.jpg diff --git a/public/logo.png b/frontend/public/logo.png similarity index 100% rename from public/logo.png rename to frontend/public/logo.png diff --git a/public/rubin.png b/frontend/public/rubin.png similarity index 100% rename from public/rubin.png rename to frontend/public/rubin.png diff --git a/services/api.js b/frontend/services/api.js similarity index 94% rename from services/api.js rename to frontend/services/api.js index 3c1943d..6d680c2 100644 --- a/services/api.js +++ b/frontend/services/api.js @@ -5,7 +5,7 @@ export function getAPIClient(ctx) { const { 'pzserver.token': token } = parseCookies(ctx) const api = axios.create({ - baseURL: '/' + baseURL: '/api' }) if (token) { diff --git a/services/auth.js b/frontend/services/auth.js similarity index 100% rename from services/auth.js rename to frontend/services/auth.js diff --git a/styles/components/Footer.js b/frontend/styles/components/Footer.js similarity index 100% rename from styles/components/Footer.js rename to frontend/styles/components/Footer.js diff --git a/styles/components/Header.js b/frontend/styles/components/Header.js similarity index 100% rename from styles/components/Header.js rename to frontend/styles/components/Header.js diff --git a/styles/global.css b/frontend/styles/global.css similarity index 100% rename from styles/global.css rename to frontend/styles/global.css diff --git a/styles/pages/index.js b/frontend/styles/pages/index.js similarity index 100% rename from styles/pages/index.js rename to frontend/styles/pages/index.js diff --git a/styles/pages/login.js b/frontend/styles/pages/login.js similarity index 100% rename from styles/pages/login.js rename to frontend/styles/pages/login.js diff --git a/styles/pages/upload.js b/frontend/styles/pages/upload.js similarity index 100% rename from styles/pages/upload.js rename to frontend/styles/pages/upload.js diff --git a/themes/light.js b/frontend/themes/light.js similarity index 100% rename from themes/light.js rename to frontend/themes/light.js diff --git a/yarn.lock b/frontend/yarn.lock similarity index 100% rename from yarn.lock rename to frontend/yarn.lock diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..1e26085 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,110 @@ +upstream pzapi { + server backend:8000; +} + +upstream pzfrontend { + server frontend:3000; +} + +server { + + listen 8080; + + client_max_body_size 200M; + + charset utf-8; + + gzip on; + gzip_comp_level 9; + gzip_min_length 128; + gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon; + + proxy_send_timeout 120s; + proxy_read_timeout 120s; + fastcgi_send_timeout 120s; + fastcgi_read_timeout 120s; + + # access_log /var/log/nginx/host.access.log main; + + # location / { + # proxy_pass http://pzfrontend$request_uri; + # proxy_set_header Host $host; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Host $server_name; + # proxy_set_header X-Real-IP $remote_addr; + # } + + # Proxy pass to frontend development server with live relead + # Based in this article: https://nathanfriend.io/2018/05/14/live-reloading-an-angular-2-app-behind-nginx.html + location / { + proxy_pass http://pzfrontend; + + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + + # live reload + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_cache_bypass $http_upgrade; + } + + + + + + + location /api { + proxy_pass http://pzapi; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + location /admin { + proxy_pass http://pzapi; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + + location /django_static { + alias /var/www/django_static/; + try_files $uri $uri/ /index.html; + autoindex off; + } + + location /archive/data { + alias /var/www/data/; + try_files $uri $uri/ /index.html; + autoindex off; + } + + +# -------------------------- + # location /archive/data/ { + # alias /archive/data/; + # try_files $uri $uri/ /index.html; + # autoindex off; + # } + + # location /static/ { + # alias /var/www/static/; + # try_files $uri $uri/ /index.html; + # autoindex off; + # } + + #location / { + # include uwsgi_params; + # uwsgi_pass pzapi; + #} + + # # Admin + # location /admin/ { + # include uwsgi_params; + # uwsgi_pass pzapi; + # } + +} \ No newline at end of file