diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8f38f04 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +.git +.gitignore +.dockerignore +Dockerfile + +.env +.venv +__pycache__ +*.py[cod] +*.sqlite3 + +.pytest_cache +.mypy_cache +.ruff_cache +.coverage +htmlcov + +staticfiles +media +docker-data + +docs diff --git a/.env.example b/.env.example index 0b57ce2..c1733dc 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,7 @@ DJANGO_DEBUG=True DJANGO_SECRET_KEY=django-insecure-dev-only-change-me -DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 +DJANGO_ALLOWED_HOSTS=autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0 +DJANGO_CSRF_TRUSTED_ORIGINS=https://autocensup.solucaoti.net.br POSTGRES_DB=autocensup POSTGRES_USER=autocensup diff --git a/.gitignore b/.gitignore index afab935..06eb365 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ wheels/ staticfiles/ media/ *.sqlite3 + +# Docker bind mounts +docker-data/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d5f14aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM ghcr.io/astral-sh/uv:0.5.11-python3.12-bookworm-slim AS builder + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-dev + +COPY . . +RUN uv run python manage.py collectstatic --noinput + +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/app/.venv/bin:$PATH" \ + DJANGO_DEBUG=False \ + PORT=8000 + +WORKDIR /app + +RUN useradd --create-home --shell /usr/sbin/nologin appuser + +COPY --from=builder /app /app + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8000 + +CMD ["sh", "-c", "python manage.py migrate --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:${PORT}"] diff --git a/Makefile b/Makefile index b9b4313..e44d0e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ -.PHONY: up down logs worker +IMAGE ?= gitea.solucaoti.net.br/rogeriolima/autocensup:latest + +.PHONY: up down logs worker build push pull up: - docker compose up -d db valkey + docker compose up -d down: docker compose down @@ -11,3 +13,12 @@ logs: worker: UV_CACHE_DIR=/tmp/uv-cache uv run celery -A config worker -l info + +build: + docker build -t $(IMAGE) . + +push: + docker push $(IMAGE) + +pull: + docker pull $(IMAGE) diff --git a/config/settings.py b/config/settings.py index d3d6e0c..47e24e8 100644 --- a/config/settings.py +++ b/config/settings.py @@ -35,10 +35,24 @@ DEBUG = os.getenv("DJANGO_DEBUG", "True").lower() in {"1", "true", "yes", "on"} ALLOWED_HOSTS = [ host.strip() - for host in os.getenv("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,0.0.0.0").split(",") + for host in os.getenv( + "DJANGO_ALLOWED_HOSTS", + "autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0", + ).split(",") if host.strip() ] +CSRF_TRUSTED_ORIGINS = [ + origin.strip() + for origin in os.getenv( + "DJANGO_CSRF_TRUSTED_ORIGINS", + "https://autocensup.solucaoti.net.br", + ).split(",") + if origin.strip() +] + +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", diff --git a/core/importers.py b/core/importers.py index e2294c7..230a434 100644 --- a/core/importers.py +++ b/core/importers.py @@ -1,6 +1,7 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor, as_completed +import io import csv from dataclasses import dataclass from datetime import date, datetime @@ -61,12 +62,24 @@ def _gennera_get_with_retry( raise last_error +@dataclass +class ImportCursoError: + line: int + identifier: str + message: str + + @dataclass class ImportCursosResult: created: int = 0 updated: int = 0 without_carga_horaria: int = 0 total: int = 0 + errors: list[ImportCursoError] | None = None + + def __post_init__(self) -> None: + if self.errors is None: + self.errors = [] @dataclass @@ -204,36 +217,72 @@ def load_max_carga_horaria_by_curso(alunos_path: Path) -> dict[int, int]: return cargas -def import_cursos(csv_path: Path, alunos_path: Path) -> ImportCursosResult: - cargas = load_max_carga_horaria_by_curso(alunos_path) +def import_cursos( + csv_path: Path | None = None, + alunos_path: Path | None = None, + csv_text: str | None = None, +) -> ImportCursosResult: + cargas = load_max_carga_horaria_by_curso(alunos_path) if alunos_path else {} result = ImportCursosResult() - with csv_path.open(encoding="utf-8-sig", newline="") as file: + if csv_text is None: + if csv_path is None: + raise ValueError("csv_path ou csv_text obrigatorio.") + csv_file = csv_path.open(encoding="utf-8-sig", newline="") + else: + csv_file = io.StringIO(csv_text) + + with csv_file as file: reader = csv.DictReader(file, delimiter=";") - for row in reader: - codigo_mec = int(row["CODIGO_CURSO"]) - carga_horaria_total = cargas.get(codigo_mec, 0) - if carga_horaria_total == 0: - result.without_carga_horaria += 1 - - _, created = Curso.objects.update_or_create( - codigo_mec=codigo_mec, - defaults={ - "descricao": row["NOME_CURSO"], - "situacao_funcionamento_curso": row[ - "SITUACAO_FUNCIONAMENTO_CURSO" - ], - "licenciatura": row["GRAU_ACADEMICO"] == "Licenciatura", - "ead": row["FORMATO_OFERTA"] == "EaD", - "carga_horaria_total": carga_horaria_total, - }, - ) - + for line_number, row in enumerate(reader, start=2): result.total += 1 - if created: - result.created += 1 - else: - result.updated += 1 + identifier = str(row.get("CODIGO_CURSO", "")).strip() + try: + codigo_mec = parse_required_int(identifier, "Codigo MEC do curso") + descricao = str(row.get("NOME_CURSO", "")).strip() + if not descricao: + raise ValueError("Nome do curso obrigatorio nao informado.") + + situacao = str(row.get("SITUACAO_FUNCIONAMENTO_CURSO", "")).strip() + formato_oferta = str(row.get("FORMATO_OFERTA", "")).strip() + grau_academico = str(row.get("GRAU_ACADEMICO", "")).strip() + exportar = "extin" not in normalize_label(situacao) + + carga_horaria_total = cargas.get(codigo_mec) + if carga_horaria_total is None: + existing = Curso.objects.filter(codigo_mec=codigo_mec).only( + "carga_horaria_total" + ).first() + carga_horaria_total = ( + existing.carga_horaria_total if existing is not None else 0 + ) + result.without_carga_horaria += 1 + + _, created = Curso.objects.update_or_create( + codigo_mec=codigo_mec, + defaults={ + "descricao": descricao, + "situacao_funcionamento_curso": situacao, + "licenciatura": normalize_label(grau_academico) + == "licenciatura", + "ead": normalize_label(formato_oferta) == "ead", + "exportar": exportar, + "carga_horaria_total": carga_horaria_total, + }, + ) + + if created: + result.created += 1 + else: + result.updated += 1 + except (ValueError, IntegrityError) as exc: + result.errors.append( + ImportCursoError( + line=line_number, + identifier=identifier, + message=str(exc), + ) + ) return result diff --git a/core/management/commands/import_cursos.py b/core/management/commands/import_cursos.py index f62f1b3..c2c5f4e 100644 --- a/core/management/commands/import_cursos.py +++ b/core/management/commands/import_cursos.py @@ -6,16 +6,21 @@ from core.importers import import_cursos class Command(BaseCommand): - help = "Importa cursos de um CSV usando carga horaria maxima do TXT de alunos." + help = "Importa cursos de um CSV no formato de docs/cursos.csv." def add_arguments(self, parser) -> None: parser.add_argument("--csv", default="docs/cursos.csv") - parser.add_argument("--alunos", default="docs/aluno_exportacao.txt") + parser.add_argument( + "--alunos", + default="docs/aluno_exportacao.txt", + help="Opcional. Usa o TXT de alunos para preencher carga horaria maxima.", + ) def handle(self, *args, **options) -> None: + alunos_path = Path(options["alunos"]) result = import_cursos( csv_path=Path(options["csv"]), - alunos_path=Path(options["alunos"]), + alunos_path=alunos_path if alunos_path.exists() else None, ) self.stdout.write( @@ -24,6 +29,9 @@ class Command(BaseCommand): f"{result.total} processados, " f"{result.created} criados, " f"{result.updated} atualizados, " - f"{result.without_carga_horaria} sem carga horaria no TXT." + f"{result.without_carga_horaria} sem carga horaria de apoio, " + f"{len(result.errors)} com erro." ) ) + for error in result.errors: + self.stdout.write(self.style.WARNING(f" Linha {error.line}: {error.message}")) diff --git a/core/tests.py b/core/tests.py index 99ad48f..1534ee1 100644 --- a/core/tests.py +++ b/core/tests.py @@ -765,6 +765,39 @@ class ImportCursosTests(TestCase): self.assertEqual(curso_criado.carga_horaria_total, 4000) self.assertEqual(curso_sem_carga.carga_horaria_total, 0) + def test_import_cursos_from_cursos_csv_without_support_file(self) -> None: + Curso.objects.create( + codigo_mec=123, + descricao="Descricao antiga", + carga_horaria_total=3200, + ) + + csv_text = ( + "\ufeffCODIGO_IES;NOME_IES;CODIGO_CURSO;NOME_CURSO;" + "CURSO_ATUALIZADO;SITUACAO_FUNCIONAMENTO_CURSO;FORMATO_OFERTA;GRAU_ACADEMICO\n" + "1131;IES;123;PEDAGOGIA;Sim;Em atividade;EaD;Licenciatura\n" + "1131;IES;456;ENGENHARIA;Sim;Em extinção;Presencial;Bacharelado\n" + ) + + result = import_cursos(csv_text=csv_text) + + curso_atualizado = Curso.objects.get(codigo_mec=123) + curso_criado = Curso.objects.get(codigo_mec=456) + + self.assertEqual(result.total, 2) + self.assertEqual(result.created, 1) + self.assertEqual(result.updated, 1) + self.assertEqual(result.without_carga_horaria, 2) + self.assertEqual(curso_atualizado.descricao, "PEDAGOGIA") + self.assertEqual(curso_atualizado.carga_horaria_total, 3200) + self.assertTrue(curso_atualizado.licenciatura) + self.assertTrue(curso_atualizado.ead) + self.assertTrue(curso_atualizado.exportar) + self.assertEqual(curso_criado.carga_horaria_total, 0) + self.assertFalse(curso_criado.licenciatura) + self.assertFalse(curso_criado.ead) + self.assertFalse(curso_criado.exportar) + def vinculo_line(self, codigo_curso: int, carga_horaria: int) -> str: fields = [""] * 73 fields[0] = "42" @@ -1061,6 +1094,32 @@ class ImportAlunosTests(AuthenticatedViewTestCase): self.assertEqual(Pessoa.objects.count(), 1) self.assertEqual(Vinculo.objects.count(), 1) + +class ImportCursosViewTests(AuthenticatedViewTestCase): + def test_import_cursos_view_accepts_upload(self) -> None: + uploaded = SimpleUploadedFile( + "cursos.csv", + ( + "\ufeffCODIGO_IES;NOME_IES;CODIGO_CURSO;NOME_CURSO;" + "CURSO_ATUALIZADO;SITUACAO_FUNCIONAMENTO_CURSO;FORMATO_OFERTA;GRAU_ACADEMICO\n" + "1131;IES;123;PEDAGOGIA;Sim;Em atividade;EaD;Licenciatura\n" + ).encode("utf-8"), + content_type="text/csv", + ) + + response = self.client.post( + reverse("import_cursos"), + {"arquivo": uploaded}, + ) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Cursos criados") + self.assertEqual(Curso.objects.count(), 1) + curso = Curso.objects.get(codigo_mec=123) + self.assertEqual(curso.descricao, "PEDAGOGIA") + self.assertTrue(curso.licenciatura) + self.assertTrue(curso.ead) + def pessoa_line(self, nome: str, nascimento: str) -> str: return "|".join( [ diff --git a/core/urls.py b/core/urls.py index c5dcdf4..e1ec4f6 100644 --- a/core/urls.py +++ b/core/urls.py @@ -9,6 +9,7 @@ from .views import ( export_alunos_view, home, import_alunos_view, + import_cursos_view, pending_required_fields, processar_situacao_view, set_ano_censo, @@ -21,6 +22,7 @@ urlpatterns = [ path("", home, name="home"), path("ano-censo/", set_ano_censo, name="set_ano_censo"), path("importar/alunos/", import_alunos_view, name="import_alunos"), + path("importar/cursos/", import_cursos_view, name="import_cursos"), path("exportar/alunos//", export_alunos_view, name="export_alunos"), path("processar/situacao/", processar_situacao_view, name="processar_situacao"), path("pendencias/", pending_required_fields, name="pending_required_fields"), diff --git a/core/views.py b/core/views.py index e238ba1..f59b1ad 100644 --- a/core/views.py +++ b/core/views.py @@ -21,7 +21,7 @@ from django.urls import reverse from django.utils.http import url_has_allowed_host_and_scheme from django.core.paginator import Paginator -from .importers import build_aluno_export_file, import_alunos +from .importers import build_aluno_export_file, import_alunos, import_cursos from .models import ( AnoCenso, Curso, @@ -193,6 +193,15 @@ class ImportAlunosForm(forms.Form): self.fields["arquivo"].widget.attrs["accept"] = ".txt,text/plain" +class ImportCursosForm(forms.Form): + arquivo = forms.FileField() + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.fields["arquivo"].widget.attrs["class"] = "form-control" + self.fields["arquivo"].widget.attrs["accept"] = ".csv,text/csv" + + class ProcessarSituacaoForm(forms.Form): matriculas = forms.CharField( label="Matricula", @@ -387,6 +396,29 @@ def import_alunos_view(request: HttpRequest) -> HttpResponse: return render( request, "core/import_alunos.html", + { + "form": form, + "result": result, + }, + ) + + +@login_required +def import_cursos_view(request: HttpRequest) -> HttpResponse: + result = None + form = ImportCursosForm(request.POST or None, request.FILES or None) + + if request.method == "POST" and form.is_valid(): + uploaded_file = form.cleaned_data["arquivo"] + content = uploaded_file.read().decode("utf-8-sig") + result = import_cursos( + csv_text=content, + ) + messages.success(request, "Importacao de cursos processada.") + + return render( + request, + "core/import_cursos.html", { "form": form, "result": result, diff --git a/docker-compose.yml b/docker-compose.yml index 1a10bfe..de782d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,57 @@ services: + app: + build: + context: . + image: gitea.solucaoti.net.br/rogeriolima/autocensup:latest + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + DJANGO_DEBUG: "False" + DJANGO_ALLOWED_HOSTS: autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0 + DJANGO_CSRF_TRUSTED_ORIGINS: https://autocensup.solucaoti.net.br + POSTGRES_DB: autocensup + POSTGRES_USER: autocensup + POSTGRES_PASSWORD: autocensup + POSTGRES_HOST: db + POSTGRES_PORT: "5432" + CELERY_BROKER_URL: redis://valkey:6379/0 + CELERY_RESULT_BACKEND: redis://valkey:6379/0 + ports: + - "8000:8000" + volumes: + - ./docker-data/media:/app/media + depends_on: + db: + condition: service_healthy + valkey: + condition: service_healthy + + worker: + image: gitea.solucaoti.net.br/rogeriolima/autocensup:latest + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + DJANGO_DEBUG: "False" + DJANGO_ALLOWED_HOSTS: autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0 + DJANGO_CSRF_TRUSTED_ORIGINS: https://autocensup.solucaoti.net.br + POSTGRES_DB: autocensup + POSTGRES_USER: autocensup + POSTGRES_PASSWORD: autocensup + POSTGRES_HOST: db + POSTGRES_PORT: "5432" + CELERY_BROKER_URL: redis://valkey:6379/0 + CELERY_RESULT_BACKEND: redis://valkey:6379/0 + command: celery -A config worker -l info + depends_on: + db: + condition: service_healthy + valkey: + condition: service_healthy + db: image: postgres:16-alpine restart: unless-stopped @@ -9,7 +62,7 @@ services: ports: - "5432:5432" volumes: - - postgres_data:/var/lib/postgresql/data + - ./docker-data/postgres:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U autocensup -d autocensup"] interval: 10s @@ -22,13 +75,9 @@ services: ports: - "6379:6379" volumes: - - valkey_data:/data + - ./docker-data/valkey:/data healthcheck: test: ["CMD", "valkey-cli", "ping"] interval: 10s timeout: 5s retries: 5 - -volumes: - postgres_data: - valkey_data: diff --git a/pyproject.toml b/pyproject.toml index 39b1e62..aef3396 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,5 +8,6 @@ dependencies = [ "django>=5.2,<6.0", "psycopg[binary]>=3.2,<4.0", "celery[redis]>=5.4,<6.0", + "gunicorn>=23.0,<24.0", "requests>=2.31,<3.0", ] diff --git a/templates/base.html b/templates/base.html index b12f3e1..fdca200 100644 --- a/templates/base.html +++ b/templates/base.html @@ -171,6 +171,10 @@ Importar Aluno do CENSUP + + + Importar Cursos + Processar Situacao diff --git a/templates/core/import_cursos.html b/templates/core/import_cursos.html new file mode 100644 index 0000000..a071184 --- /dev/null +++ b/templates/core/import_cursos.html @@ -0,0 +1,89 @@ +{% extends "base.html" %} + +{% block title %}Importar Cursos | Autocensup{% endblock %} +{% block page_pretitle %}Importacao{% endblock %} +{% block page_title %}Importar Cursos{% endblock %} + +{% block content %} +
+
+
+ {% csrf_token %} +
+ {% if form.non_field_errors %} +
{{ form.non_field_errors }}
+ {% endif %} +
+ + {{ form.arquivo }} + {% for error in form.arquivo.errors %} +
{{ error }}
+ {% endfor %} +
+
+
+ +
+
+
+ + {% if result %} +
+
+
+

Resultado

+
+
+
+
+
Linhas processadas
+
{{ result.total }}
+
+
+
Cursos criados
+
{{ result.created }}
+
+
+
Cursos atualizados
+
{{ result.updated }}
+
+
+
Sem carga de apoio
+
{{ result.without_carga_horaria }}
+
+
+
+
+ + + + + + + + + + {% for error in result.errors %} + + + + + + {% empty %} + + + + {% endfor %} + +
LinhaCodigoErro
{{ error.line }}{{ error.identifier|default:"-" }}{{ error.message }}
+ Nenhum erro encontrado. +
+
+
+
+ {% endif %} +
+{% endblock %} diff --git a/uv.lock b/uv.lock index 2b686e3..fb41292 100644 --- a/uv.lock +++ b/uv.lock @@ -30,6 +30,7 @@ source = { virtual = "." } dependencies = [ { name = "celery", extra = ["redis"] }, { name = "django" }, + { name = "gunicorn" }, { name = "psycopg", extra = ["binary"] }, { name = "requests" }, ] @@ -38,6 +39,7 @@ dependencies = [ requires-dist = [ { name = "celery", extras = ["redis"], specifier = ">=5.4,<6.0" }, { name = "django", specifier = ">=5.2,<6.0" }, + { name = "gunicorn", specifier = ">=23.0,<24.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4.0" }, { name = "requests", specifier = ">=2.31,<3.0" }, ] @@ -218,6 +220,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/44/f172870cf87aa25afef48fb72adba89ee8b77fcab6f3b23d240b923f1528/django-5.2.14-py3-none-any.whl", hash = "sha256:6f712143bd3064310d1f50fac859c3e9a274bdcfc9595339853be7779297fc76", size = 8311320, upload-time = "2026-05-05T13:57:25.795Z" }, ] +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, +] + [[package]] name = "idna" version = "3.18"