feat(import): add cursos upload flow

.dockerignore

@@ -0,0 +1,22 @@
+.git
+.gitignore
+.dockerignore
+Dockerfile
+
+.env
+.venv
+__pycache__
+*.py[cod]
+*.sqlite3
+
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.coverage
+htmlcov
+
+staticfiles
+media
+docker-data
+
+docs

.env.example

@@ -1,6 +1,7 @@
 DJANGO_DEBUG=True
 DJANGO_SECRET_KEY=django-insecure-dev-only-change-me
-DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0
+DJANGO_ALLOWED_HOSTS=autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0
+DJANGO_CSRF_TRUSTED_ORIGINS=https://autocensup.solucaoti.net.br
 
 POSTGRES_DB=autocensup
 POSTGRES_USER=autocensup

.gitignore

@@ -16,3 +16,6 @@ wheels/
 staticfiles/
 media/
 *.sqlite3
+
+# Docker bind mounts
+docker-data/

Dockerfile

@@ -0,0 +1,34 @@
+FROM ghcr.io/astral-sh/uv:0.5.11-python3.12-bookworm-slim AS builder
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev
+
+COPY . .
+RUN uv run python manage.py collectstatic --noinput
+
+FROM python:3.12-slim-bookworm
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/app/.venv/bin:$PATH" \
+    DJANGO_DEBUG=False \
+    PORT=8000
+
+WORKDIR /app
+
+RUN useradd --create-home --shell /usr/sbin/nologin appuser
+
+COPY --from=builder /app /app
+
+RUN chown -R appuser:appuser /app
+
+USER appuser
+
+EXPOSE 8000
+
+CMD ["sh", "-c", "python manage.py migrate --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:${PORT}"]

Makefile

@@ -1,7 +1,9 @@
-.PHONY: up down logs worker
+IMAGE ?= gitea.solucaoti.net.br/rogeriolima/autocensup:latest
+
+.PHONY: up down logs worker build push pull
 
 up:
-	docker compose up -d db valkey
+	docker compose up -d
 
 down:
 	docker compose down
@@ -11,3 +13,12 @@ logs:
 
 worker:
 	UV_CACHE_DIR=/tmp/uv-cache uv run celery -A config worker -l info
+
+build:
+	docker build -t $(IMAGE) .
+
+push:
+	docker push $(IMAGE)
+
+pull:
+	docker pull $(IMAGE)

config/settings.py

@@ -35,10 +35,24 @@ DEBUG = os.getenv("DJANGO_DEBUG", "True").lower() in {"1", "true", "yes", "on"}
 
 ALLOWED_HOSTS = [
     host.strip()
-    for host in os.getenv("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,0.0.0.0").split(",")
+    for host in os.getenv(
+        "DJANGO_ALLOWED_HOSTS",
+        "autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0",
+    ).split(",")
     if host.strip()
 ]
 
+CSRF_TRUSTED_ORIGINS = [
+    origin.strip()
+    for origin in os.getenv(
+        "DJANGO_CSRF_TRUSTED_ORIGINS",
+        "https://autocensup.solucaoti.net.br",
+    ).split(",")
+    if origin.strip()
+]
+
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
 INSTALLED_APPS = [
     "django.contrib.admin",
     "django.contrib.auth",

core/importers.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
+import io
 import csv
 from dataclasses import dataclass
 from datetime import date, datetime
@@ -61,12 +62,24 @@ def _gennera_get_with_retry(
     raise last_error
 
 
+@dataclass
+class ImportCursoError:
+    line: int
+    identifier: str
+    message: str
+
+
 @dataclass
 class ImportCursosResult:
     created: int = 0
     updated: int = 0
     without_carga_horaria: int = 0
     total: int = 0
+    errors: list[ImportCursoError] | None = None
+
+    def __post_init__(self) -> None:
+        if self.errors is None:
+            self.errors = []
 
 
 @dataclass
@@ -204,36 +217,72 @@ def load_max_carga_horaria_by_curso(alunos_path: Path) -> dict[int, int]:
     return cargas
 
 
-def import_cursos(csv_path: Path, alunos_path: Path) -> ImportCursosResult:
+def import_cursos(
-    cargas = load_max_carga_horaria_by_curso(alunos_path)
+    csv_path: Path | None = None,
+    alunos_path: Path | None = None,
+    csv_text: str | None = None,
+) -> ImportCursosResult:
+    cargas = load_max_carga_horaria_by_curso(alunos_path) if alunos_path else {}
     result = ImportCursosResult()
 
-    with csv_path.open(encoding="utf-8-sig", newline="") as file:
+    if csv_text is None:
+        if csv_path is None:
+            raise ValueError("csv_path ou csv_text obrigatorio.")
+        csv_file = csv_path.open(encoding="utf-8-sig", newline="")
+    else:
+        csv_file = io.StringIO(csv_text)
+
+    with csv_file as file:
         reader = csv.DictReader(file, delimiter=";")
-        for row in reader:
+        for line_number, row in enumerate(reader, start=2):
-            codigo_mec = int(row["CODIGO_CURSO"])
+            result.total += 1
-            carga_horaria_total = cargas.get(codigo_mec, 0)
+            identifier = str(row.get("CODIGO_CURSO", "")).strip()
-            if carga_horaria_total == 0:
+            try:
+                codigo_mec = parse_required_int(identifier, "Codigo MEC do curso")
+                descricao = str(row.get("NOME_CURSO", "")).strip()
+                if not descricao:
+                    raise ValueError("Nome do curso obrigatorio nao informado.")
+
+                situacao = str(row.get("SITUACAO_FUNCIONAMENTO_CURSO", "")).strip()
+                formato_oferta = str(row.get("FORMATO_OFERTA", "")).strip()
+                grau_academico = str(row.get("GRAU_ACADEMICO", "")).strip()
+                exportar = "extin" not in normalize_label(situacao)
+
+                carga_horaria_total = cargas.get(codigo_mec)
+                if carga_horaria_total is None:
+                    existing = Curso.objects.filter(codigo_mec=codigo_mec).only(
+                        "carga_horaria_total"
+                    ).first()
+                    carga_horaria_total = (
+                        existing.carga_horaria_total if existing is not None else 0
+                    )
                     result.without_carga_horaria += 1
 
                 _, created = Curso.objects.update_or_create(
                     codigo_mec=codigo_mec,
                     defaults={
-                    "descricao": row["NOME_CURSO"],
+                        "descricao": descricao,
-                    "situacao_funcionamento_curso": row[
+                        "situacao_funcionamento_curso": situacao,
-                        "SITUACAO_FUNCIONAMENTO_CURSO"
+                        "licenciatura": normalize_label(grau_academico)
-                    ],
+                        == "licenciatura",
-                    "licenciatura": row["GRAU_ACADEMICO"] == "Licenciatura",
+                        "ead": normalize_label(formato_oferta) == "ead",
-                    "ead": row["FORMATO_OFERTA"] == "EaD",
+                        "exportar": exportar,
                         "carga_horaria_total": carga_horaria_total,
                     },
                 )
 
-            result.total += 1
                 if created:
                     result.created += 1
                 else:
                     result.updated += 1
+            except (ValueError, IntegrityError) as exc:
+                result.errors.append(
+                    ImportCursoError(
+                        line=line_number,
+                        identifier=identifier,
+                        message=str(exc),
+                    )
+                )
 
     return result
 

core/management/commands/import_cursos.py

@@ -6,16 +6,21 @@ from core.importers import import_cursos
 
 
 class Command(BaseCommand):
-    help = "Importa cursos de um CSV usando carga horaria maxima do TXT de alunos."
+    help = "Importa cursos de um CSV no formato de docs/cursos.csv."
 
     def add_arguments(self, parser) -> None:
         parser.add_argument("--csv", default="docs/cursos.csv")
-        parser.add_argument("--alunos", default="docs/aluno_exportacao.txt")
+        parser.add_argument(
+            "--alunos",
+            default="docs/aluno_exportacao.txt",
+            help="Opcional. Usa o TXT de alunos para preencher carga horaria maxima.",
+        )
 
     def handle(self, *args, **options) -> None:
+        alunos_path = Path(options["alunos"])
         result = import_cursos(
             csv_path=Path(options["csv"]),
-            alunos_path=Path(options["alunos"]),
+            alunos_path=alunos_path if alunos_path.exists() else None,
         )
 
         self.stdout.write(
@@ -24,6 +29,9 @@ class Command(BaseCommand):
                 f"{result.total} processados, "
                 f"{result.created} criados, "
                 f"{result.updated} atualizados, "
-                f"{result.without_carga_horaria} sem carga horaria no TXT."
+                f"{result.without_carga_horaria} sem carga horaria de apoio, "
+                f"{len(result.errors)} com erro."
             )
         )
+        for error in result.errors:
+            self.stdout.write(self.style.WARNING(f"  Linha {error.line}: {error.message}"))

core/tests.py

@@ -765,6 +765,39 @@ class ImportCursosTests(TestCase):
         self.assertEqual(curso_criado.carga_horaria_total, 4000)
         self.assertEqual(curso_sem_carga.carga_horaria_total, 0)
 
+    def test_import_cursos_from_cursos_csv_without_support_file(self) -> None:
+        Curso.objects.create(
+            codigo_mec=123,
+            descricao="Descricao antiga",
+            carga_horaria_total=3200,
+        )
+
+        csv_text = (
+            "\ufeffCODIGO_IES;NOME_IES;CODIGO_CURSO;NOME_CURSO;"
+            "CURSO_ATUALIZADO;SITUACAO_FUNCIONAMENTO_CURSO;FORMATO_OFERTA;GRAU_ACADEMICO\n"
+            "1131;IES;123;PEDAGOGIA;Sim;Em atividade;EaD;Licenciatura\n"
+            "1131;IES;456;ENGENHARIA;Sim;Em extinção;Presencial;Bacharelado\n"
+        )
+
+        result = import_cursos(csv_text=csv_text)
+
+        curso_atualizado = Curso.objects.get(codigo_mec=123)
+        curso_criado = Curso.objects.get(codigo_mec=456)
+
+        self.assertEqual(result.total, 2)
+        self.assertEqual(result.created, 1)
+        self.assertEqual(result.updated, 1)
+        self.assertEqual(result.without_carga_horaria, 2)
+        self.assertEqual(curso_atualizado.descricao, "PEDAGOGIA")
+        self.assertEqual(curso_atualizado.carga_horaria_total, 3200)
+        self.assertTrue(curso_atualizado.licenciatura)
+        self.assertTrue(curso_atualizado.ead)
+        self.assertTrue(curso_atualizado.exportar)
+        self.assertEqual(curso_criado.carga_horaria_total, 0)
+        self.assertFalse(curso_criado.licenciatura)
+        self.assertFalse(curso_criado.ead)
+        self.assertFalse(curso_criado.exportar)
+
     def vinculo_line(self, codigo_curso: int, carga_horaria: int) -> str:
         fields = [""] * 73
         fields[0] = "42"
@@ -1061,6 +1094,32 @@ class ImportAlunosTests(AuthenticatedViewTestCase):
         self.assertEqual(Pessoa.objects.count(), 1)
         self.assertEqual(Vinculo.objects.count(), 1)
 
+
+class ImportCursosViewTests(AuthenticatedViewTestCase):
+    def test_import_cursos_view_accepts_upload(self) -> None:
+        uploaded = SimpleUploadedFile(
+            "cursos.csv",
+            (
+                "\ufeffCODIGO_IES;NOME_IES;CODIGO_CURSO;NOME_CURSO;"
+                "CURSO_ATUALIZADO;SITUACAO_FUNCIONAMENTO_CURSO;FORMATO_OFERTA;GRAU_ACADEMICO\n"
+                "1131;IES;123;PEDAGOGIA;Sim;Em atividade;EaD;Licenciatura\n"
+            ).encode("utf-8"),
+            content_type="text/csv",
+        )
+
+        response = self.client.post(
+            reverse("import_cursos"),
+            {"arquivo": uploaded},
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, "Cursos criados")
+        self.assertEqual(Curso.objects.count(), 1)
+        curso = Curso.objects.get(codigo_mec=123)
+        self.assertEqual(curso.descricao, "PEDAGOGIA")
+        self.assertTrue(curso.licenciatura)
+        self.assertTrue(curso.ead)
+
     def pessoa_line(self, nome: str, nascimento: str) -> str:
         return "|".join(
             [

core/urls.py

@@ -9,6 +9,7 @@ from .views import (
     export_alunos_view,
     home,
     import_alunos_view,
+    import_cursos_view,
     pending_required_fields,
     processar_situacao_view,
     set_ano_censo,
@@ -21,6 +22,7 @@ urlpatterns = [
     path("", home, name="home"),
     path("ano-censo/", set_ano_censo, name="set_ano_censo"),
     path("importar/alunos/", import_alunos_view, name="import_alunos"),
+    path("importar/cursos/", import_cursos_view, name="import_cursos"),
     path("exportar/alunos/<slug:filtro>/", export_alunos_view, name="export_alunos"),
     path("processar/situacao/", processar_situacao_view, name="processar_situacao"),
     path("pendencias/", pending_required_fields, name="pending_required_fields"),

core/views.py

@@ -21,7 +21,7 @@ from django.urls import reverse
 from django.utils.http import url_has_allowed_host_and_scheme
 from django.core.paginator import Paginator
 
-from .importers import build_aluno_export_file, import_alunos
+from .importers import build_aluno_export_file, import_alunos, import_cursos
 from .models import (
     AnoCenso,
     Curso,
@@ -193,6 +193,15 @@ class ImportAlunosForm(forms.Form):
         self.fields["arquivo"].widget.attrs["accept"] = ".txt,text/plain"
 
 
+class ImportCursosForm(forms.Form):
+    arquivo = forms.FileField()
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.fields["arquivo"].widget.attrs["class"] = "form-control"
+        self.fields["arquivo"].widget.attrs["accept"] = ".csv,text/csv"
+
+
 class ProcessarSituacaoForm(forms.Form):
     matriculas = forms.CharField(
         label="Matricula",
@@ -394,6 +403,29 @@ def import_alunos_view(request: HttpRequest) -> HttpResponse:
     )
 
 
+@login_required
+def import_cursos_view(request: HttpRequest) -> HttpResponse:
+    result = None
+    form = ImportCursosForm(request.POST or None, request.FILES or None)
+
+    if request.method == "POST" and form.is_valid():
+        uploaded_file = form.cleaned_data["arquivo"]
+        content = uploaded_file.read().decode("utf-8-sig")
+        result = import_cursos(
+            csv_text=content,
+        )
+        messages.success(request, "Importacao de cursos processada.")
+
+    return render(
+        request,
+        "core/import_cursos.html",
+        {
+            "form": form,
+            "result": result,
+        },
+    )
+
+
 def is_task_active(task_id: str) -> bool:
     if not task_id:
         return False

docker-compose.yml

@@ -1,4 +1,57 @@
 services:
+  app:
+    build:
+      context: .
+    image: gitea.solucaoti.net.br/rogeriolima/autocensup:latest
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      DJANGO_DEBUG: "False"
+      DJANGO_ALLOWED_HOSTS: autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0
+      DJANGO_CSRF_TRUSTED_ORIGINS: https://autocensup.solucaoti.net.br
+      POSTGRES_DB: autocensup
+      POSTGRES_USER: autocensup
+      POSTGRES_PASSWORD: autocensup
+      POSTGRES_HOST: db
+      POSTGRES_PORT: "5432"
+      CELERY_BROKER_URL: redis://valkey:6379/0
+      CELERY_RESULT_BACKEND: redis://valkey:6379/0
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./docker-data/media:/app/media
+    depends_on:
+      db:
+        condition: service_healthy
+      valkey:
+        condition: service_healthy
+
+  worker:
+    image: gitea.solucaoti.net.br/rogeriolima/autocensup:latest
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      DJANGO_DEBUG: "False"
+      DJANGO_ALLOWED_HOSTS: autocensup.solucaoti.net.br,localhost,127.0.0.1,0.0.0.0
+      DJANGO_CSRF_TRUSTED_ORIGINS: https://autocensup.solucaoti.net.br
+      POSTGRES_DB: autocensup
+      POSTGRES_USER: autocensup
+      POSTGRES_PASSWORD: autocensup
+      POSTGRES_HOST: db
+      POSTGRES_PORT: "5432"
+      CELERY_BROKER_URL: redis://valkey:6379/0
+      CELERY_RESULT_BACKEND: redis://valkey:6379/0
+    command: celery -A config worker -l info
+    depends_on:
+      db:
+        condition: service_healthy
+      valkey:
+        condition: service_healthy
+
   db:
     image: postgres:16-alpine
     restart: unless-stopped
@@ -9,7 +62,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - postgres_data:/var/lib/postgresql/data
+      - ./docker-data/postgres:/var/lib/postgresql/data
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U autocensup -d autocensup"]
       interval: 10s
@@ -22,13 +75,9 @@ services:
     ports:
       - "6379:6379"
     volumes:
-      - valkey_data:/data
+      - ./docker-data/valkey:/data
     healthcheck:
       test: ["CMD", "valkey-cli", "ping"]
       interval: 10s
       timeout: 5s
       retries: 5
-
-volumes:
-  postgres_data:
-  valkey_data:

pyproject.toml

@@ -8,5 +8,6 @@ dependencies = [
     "django>=5.2,<6.0",
     "psycopg[binary]>=3.2,<4.0",
     "celery[redis]>=5.4,<6.0",
+    "gunicorn>=23.0,<24.0",
     "requests>=2.31,<3.0",
 ]

templates/base.html

@@ -171,6 +171,10 @@
                       <i class="ti ti-file-upload me-2"></i>
                       Importar Aluno do CENSUP
                     </a>
+                    <a class="dropdown-item" href="{% url 'import_cursos' %}">
+                      <i class="ti ti-school me-2"></i>
+                      Importar Cursos
+                    </a>
                     <a class="dropdown-item" href="{% url 'processar_situacao' %}">
                       <i class="ti ti-list-check me-2"></i>
                       Processar Situacao

templates/core/import_cursos.html

@@ -0,0 +1,89 @@
+{% extends "base.html" %}
+
+{% block title %}Importar Cursos | Autocensup{% endblock %}
+{% block page_pretitle %}Importacao{% endblock %}
+{% block page_title %}Importar Cursos{% endblock %}
+
+{% block content %}
+  <div class="row row-cards">
+    <div class="col-lg-5">
+      <form method="post" enctype="multipart/form-data" class="card">
+        {% csrf_token %}
+        <div class="card-body">
+          {% if form.non_field_errors %}
+            <div class="alert alert-danger">{{ form.non_field_errors }}</div>
+          {% endif %}
+          <div class="mb-3">
+            <label class="form-label" for="{{ form.arquivo.id_for_label }}">Arquivo CSV</label>
+            {{ form.arquivo }}
+            {% for error in form.arquivo.errors %}
+              <div class="invalid-feedback d-block">{{ error }}</div>
+            {% endfor %}
+          </div>
+        </div>
+        <div class="card-footer text-end">
+          <button type="submit" class="btn btn-primary">
+            <i class="ti ti-upload me-2"></i>
+            Importar
+          </button>
+        </div>
+      </form>
+    </div>
+
+    {% if result %}
+      <div class="col-lg-7">
+        <div class="card">
+          <div class="card-header">
+            <h2 class="card-title">Resultado</h2>
+          </div>
+          <div class="card-body">
+            <div class="row g-3">
+              <div class="col-sm-6 col-xl-3">
+                <div class="subheader">Linhas processadas</div>
+                <div class="h1">{{ result.total }}</div>
+              </div>
+              <div class="col-sm-6 col-xl-3">
+                <div class="subheader">Cursos criados</div>
+                <div class="h1">{{ result.created }}</div>
+              </div>
+              <div class="col-sm-6 col-xl-3">
+                <div class="subheader">Cursos atualizados</div>
+                <div class="h1">{{ result.updated }}</div>
+              </div>
+              <div class="col-sm-6 col-xl-3">
+                <div class="subheader">Sem carga de apoio</div>
+                <div class="h1">{{ result.without_carga_horaria }}</div>
+              </div>
+            </div>
+          </div>
+          <div class="table-responsive">
+            <table class="table table-vcenter card-table">
+              <thead>
+                <tr>
+                  <th>Linha</th>
+                  <th>Codigo</th>
+                  <th>Erro</th>
+                </tr>
+              </thead>
+              <tbody>
+                {% for error in result.errors %}
+                  <tr>
+                    <td>{{ error.line }}</td>
+                    <td>{{ error.identifier|default:"-" }}</td>
+                    <td>{{ error.message }}</td>
+                  </tr>
+                {% empty %}
+                  <tr>
+                    <td colspan="3" class="text-center text-secondary">
+                      Nenhum erro encontrado.
+                    </td>
+                  </tr>
+                {% endfor %}
+              </tbody>
+            </table>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+  </div>
+{% endblock %}

uv.lock

@@ -30,6 +30,7 @@ source = { virtual = "." }
 dependencies = [
     { name = "celery", extra = ["redis"] },
     { name = "django" },
+    { name = "gunicorn" },
     { name = "psycopg", extra = ["binary"] },
     { name = "requests" },
 ]
@@ -38,6 +39,7 @@ dependencies = [
 requires-dist = [
     { name = "celery", extras = ["redis"], specifier = ">=5.4,<6.0" },
     { name = "django", specifier = ">=5.2,<6.0" },
+    { name = "gunicorn", specifier = ">=23.0,<24.0" },
     { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4.0" },
     { name = "requests", specifier = ">=2.31,<3.0" },
 ]
@@ -218,6 +220,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/14/44/f172870cf87aa25afef48fb72adba89ee8b77fcab6f3b23d240b923f1528/django-5.2.14-py3-none-any.whl", hash = "sha256:6f712143bd3064310d1f50fac859c3e9a274bdcfc9595339853be7779297fc76", size = 8311320, upload-time = "2026-05-05T13:57:25.795Z" },
 ]
 
+[[package]]
+name = "gunicorn"
+version = "23.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "packaging" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" },
+]
+
 [[package]]
 name = "idna"
 version = "3.18"