Merge pull request #352 from RJ-SMTR/feat/job-transacao-view

(prod) feat: update TransacaoView

src/bigquery/repositories/bigquery-transacao.repository.ts

@@ -86,7 +86,6 @@ export class BigqueryTransacaoRepository {
           t.id_operadora,
           o.documento AS operadoraCpfCnpj,
           c.cnpj AS consorcioCnpj,
-          (${qArgs.countQuery}) AS count,
           'ok' AS status
         FROM \`${qArgs.transacao}\` t\n
         LEFT JOIN \`rj-smtr.cadastro.operadoras\` o ON o.id_operadora = t.id_operadora
@@ -95,7 +94,9 @@ export class BigqueryTransacaoRepository {
       '\n' +
       qArgs.joinIntegracao +
       '\n' +
-      (qArgs.qWhere.length ? `WHERE ${qArgs.qWhere}\n` : '') +
+      (qArgs.qWhere.length
+        ? `WHERE (${qArgs.qWhere} AND o.documento = '59675748753')\n`
+        : '') +
       `\nORDER BY datetime_processamento DESC` +
       (qArgs?.limit !== undefined ? `\nLIMIT ${qArgs.limit + 1}` : '') +
       (qArgs?.offset !== undefined ? `\nOFFSET ${qArgs.offset}` : '');
@@ -108,10 +109,8 @@ export class BigqueryTransacaoRepository {
     // Remove unwanted keys and remove last item (all null if empty)
     let transacoes: BigqueryTransacao[] = queryResult.map((i) => {
       delete i.status;
-      delete i.count;
       return i;
     });
-    transacoes.pop();
     transacoes = this.mapBqTransacao(transacoes);
 
     return {

src/bigquery/services/bigquery-ordem-pagamento.service.ts

@@ -16,28 +16,33 @@ export class BigqueryOrdemPagamentoService {
   /**
    * Get data from current payment week (qui-qua). Also with older days.
    */
-  public async getFromWeek(dataOrdemInicial,dataOrdemFinal,daysBefore=0): Promise<BigqueryOrdemPagamentoDTO[]> {
+  public async getFromWeek(
+    dataOrdemInicial: Date,
+    dataOrdemFinal: Date,
+    daysBefore = 0,
+  ): Promise<BigqueryOrdemPagamentoDTO[]> {
     const today = new Date();
-    let startDate;
-    let endDate;
+    let startDate: Date;
+    let endDate: Date;
 
-    if(dataOrdemInicial != undefined && dataOrdemFinal!=undefined) {
+    if (dataOrdemInicial != undefined && dataOrdemFinal != undefined) {
       startDate = new Date(dataOrdemInicial);
       endDate = new Date(dataOrdemFinal);
-    }else if(dataOrdemInicial != undefined && dataOrdemFinal==undefined){
+    } else if (dataOrdemInicial != undefined && dataOrdemFinal == undefined) {
       startDate = new Date(dataOrdemInicial);
       endDate = new Date(dataOrdemInicial);
-    }else{ //Sexta a Quinta
+    } else {
+      //Sexta a Quinta
       const friday = isFriday(today) ? today : nextFriday(today);
-      startDate = subDays(friday,7 + daysBefore);
-      endDate = subDays(friday,2);
+      startDate = subDays(friday, 7 + daysBefore);
+      endDate = subDays(friday, 2);
     }
     const ordemPgto = (
       await this.bigqueryOrdemPagamentoRepository.findMany({
         startDate: startDate,
-        endDate: endDate
+        endDate: endDate,
       })
     ).map((i) => ({ ...i } as BigqueryOrdemPagamentoDTO));
     return ordemPgto;
   }
-}
+}

src/bigquery/services/bigquery-transacao.service.ts

@@ -18,21 +18,25 @@ export class BigqueryTransacaoService {
    *
    * @param [daysBack=0] Pega a semana atual ou N dias atrás.
    */
-  public async getFromWeek(dataOrdemInicial,dataOrdemFinal,daysBack = 0): Promise<BigqueryTransacao[]> {
-    let startDate;
-    let endDate;
+  public async getFromWeek(
+    dataOrdemInicial: Date,
+    dataOrdemFinal: Date,
+    daysBack = 0,
+  ): Promise<BigqueryTransacao[]> {
+    let startDate: Date;
+    let endDate: Date;
     const today = new Date();
-    if(dataOrdemInicial != undefined && dataOrdemFinal !=undefined){
-      startDate = subDays(new Date(dataOrdemInicial),1);
-      endDate = subDays(new Date(dataOrdemFinal),1);
-    }else if((dataOrdemInicial != undefined) && dataOrdemFinal == undefined){
-      startDate = subDays(new Date(dataOrdemInicial),1);
-      endDate = subDays(new Date(dataOrdemInicial),1);
-    }else{
+    if (dataOrdemInicial != undefined && dataOrdemFinal != undefined) {
+      startDate = subDays(new Date(dataOrdemInicial), 1);
+      endDate = subDays(new Date(dataOrdemFinal), 1);
+    } else if (dataOrdemInicial != undefined && dataOrdemFinal == undefined) {
+      startDate = subDays(new Date(dataOrdemInicial), 1);
+      endDate = subDays(new Date(dataOrdemInicial), 1);
+    } else {
       const friday = isFriday(today) ? today : nextFriday(today);
-      startDate = subDays(friday,8 + daysBack);
-      endDate = subDays(friday,2);
-   }
+      startDate = subDays(friday, 8 + daysBack);
+      endDate = subDays(friday, 2);
+    }
     const transacao = (
       await this.bigqueryTransacaoRepository.findMany({
         startDate: startDate,

src/cnab/cnab.controller.ts

@@ -19,6 +19,7 @@ import { ExtratoDto } from './service/dto/extrato.dto';
 import { ExtratoHeaderArquivoService } from './service/extrato/extrato-header-arquivo.service';
 import { PagadorContaEnum } from './enums/pagamento/pagador.enum';
 import { CnabService } from './cnab.service';
+import { startOfDay } from 'date-fns';
 
 @ApiTags('Cnab')
 @Controller({

src/cnab/cnab.service.ts

@@ -18,11 +18,11 @@ import { SftpService } from 'src/sftp/sftp.service';
 import { TransacaoView } from 'src/transacao-bq/transacao-view.entity';
 import { TransacaoViewService } from 'src/transacao-bq/transacao-view.service';
 import { UsersService } from 'src/users/users.service';
-import { forChunk } from 'src/utils/array-utils';
+import { forChunk, getChunks } from 'src/utils/array-utils';
 import { CustomLogger } from 'src/utils/custom-logger';
 import { yearMonthDayToDate } from 'src/utils/date-utils';
 import { asNumber } from 'src/utils/pipe-utils';
-import { Between, DeepPartial } from 'typeorm';
+import { Between, DeepPartial, In } from 'typeorm';
 import { ArquivoPublicacao } from './entity/arquivo-publicacao.entity';
 import { ClienteFavorecido } from './entity/cliente-favorecido.entity';
 import { ItemTransacaoAgrupado } from './entity/pagamento/item-transacao-agrupado.entity';
@@ -59,6 +59,7 @@ import { CnabRegistros104Pgto } from './interfaces/cnab-240/104/pagamento/cnab-r
 import { CnabHeaderArquivo104 } from './interfaces/cnab-240/104/cnab-header-arquivo-104.interface';
 import { HeaderArquivoDTO } from './dto/pagamento/header-arquivo.dto';
 import { completeCPFCharacter } from 'src/utils/cpf-cnpj';
+import { ConsorcioType } from './types/consorcio.type';
 
 /**
  * User cases for CNAB and Payments
@@ -110,19 +111,23 @@ export class CnabService {
     daysBefore = 0,
     consorcio: string,
   ) {
+    const dataOrdemInicialDate = startOfDay(new Date(dataOrdemIncial));
+    const dataOrdemFinalDate = endOfDay(new Date(dataOrdemFinal));
+    const consorcioType = consorcio as ConsorcioType;
+
     // 1. Update cliente favorecido
     await this.updateAllFavorecidosFromUsers();
     // 2. Update TransacaoView
     await this.updateTransacaoViewBigquery(
-      dataOrdemIncial,
-      dataOrdemFinal,
+      dataOrdemInicialDate,
+      dataOrdemFinalDate,
       daysBefore,
-      consorcio,
+      consorcioType,
     );
     // 3. Update ordens
     const ordens = await this.bigqueryOrdemPagamentoService.getFromWeek(
-      dataOrdemIncial,
-      dataOrdemFinal,
+      dataOrdemInicialDate,
+      dataOrdemFinalDate,
       daysBefore,
     );
     await this.saveOrdens(ordens, consorcio);
@@ -132,11 +137,12 @@ export class CnabService {
    * Atualiza a tabela TransacaoView
    */
   async updateTransacaoViewBigquery(
-    dataOrdemIncial,
-    dataOrdemFinal,
+    dataOrdemIncial: Date,
+    dataOrdemFinal: Date,
     daysBack = 0,
-    consorcio: string,
+    consorcio: ConsorcioType = 'Todos',
   ) {
+    const METHOD = this.updateTransacaoViewBigquery.name;
     const transacoesBq = await this.bigqueryTransacaoService.getFromWeek(
       dataOrdemIncial,
       dataOrdemFinal,
@@ -156,7 +162,11 @@ export class CnabService {
       trs = transacoesBq.filter((tr) => tr.consorcio == consorcio);
     }
 
-    forChunk(trs, 1000, async (chunk) => {
+    const chunks = getChunks(trs, 1000);
+    let iChunk = 0;
+    for (const chunk of chunks) {
+      iChunk += 1;
+      this.logger.log(`Atualizando ${iChunk}/${chunks.length} chunks`, METHOD);
       const transacoes = chunk.map((i) =>
         TransacaoView.fromBigqueryTransacao(i),
       );
@@ -169,9 +179,10 @@ export class CnabService {
         ) {
           continue;
         }
-        await this.transacaoViewService.save(tr);
+        const existing = await this.transacaoViewService.find({ idTransacao: tr.idTransacao });
+        await this.transacaoViewService.saveMany(existing, [tr]);
       }
-    });
+    }
   }
 
   /**

src/cnab/types/cnab/cnab-file-base.type.ts

@@ -0,0 +1,4 @@
+import { CnabFile104 } from '../../interfaces/cnab-240/104/cnab-file-104.interface';
+import { CnabFile } from '../../interfaces/cnab-all/cnab-file.interface';
+
+export type CnabFileBase = CnabFile | CnabFile104;

src/cnab/types/consorcio.type.ts

@@ -0,0 +1 @@
+export type ConsorcioType = 'Van' | 'Empresa' | 'Todos';

src/cnab/utils/cnab/cnab-field-utils.ts

@@ -12,7 +12,7 @@ import {
   CnabField,
   CnabFieldFormat,
 } from '../../interfaces/cnab-all/cnab-field.interface';
-import { CnabFieldType } from '../../types/cnab-field-type.type';
+import { CnabFieldType } from '../../types/cnab/cnab-field-type.type';
 import { getCnabFieldNameLog } from './cnab-metadata-utils';
 
 type CropFillOnCrop = 'error' | 'cropLeft' | 'cropRight';

src/cnab/utils/cnab/cnab-metadata-utils.ts

@@ -1,6 +1,9 @@
-import { CnabFile104 } from "../../interfaces/cnab-240/104/cnab-file-104.interface";
-import { CnabField, CnabFields } from "../../interfaces/cnab-all/cnab-field.interface";
-import { CnabFileBase } from "../../types/cnab-file-base.type";
+import { CnabFile104 } from '../../interfaces/cnab-240/104/cnab-file-104.interface';
+import {
+  CnabField,
+  CnabFields,
+} from '../../interfaces/cnab-all/cnab-field.interface';
+import { CnabFileBase } from '../../types/cnab/cnab-file-base.type';
 
 const addMeta = addCnabTemplateMetadata;
 
@@ -11,31 +14,79 @@ export function setCnabFileMetadata(cnab: CnabFileBase, cnabName?: string) {
   cnab.headerArquivo = addMeta(cnab.headerArquivo, 'headerArquivo', cnabName);
   for (const i in cnab.lotes) {
     const lote = cnab.lotes[i];
-    lote.headerLote = addMeta(lote.headerLote, 'headerLote', cnabName, Number(i));
+    lote.headerLote = addMeta(
+      lote.headerLote,
+      'headerLote',
+      cnabName,
+      Number(i),
+    );
     for (const registro of lote.registros) {
       for (const detalheName of Object.keys(registro)) {
         const detalhe = registro[detalheName];
-        registro[detalheName] = addMeta(detalhe, detalheName, cnabName, Number(i));
+        registro[detalheName] = addMeta(
+          detalhe,
+          detalheName,
+          cnabName,
+          Number(i),
+        );
       }
     }
-    lote.trailerLote = addMeta(lote.trailerLote, 'trailerLote', cnabName, Number(i));
+    lote.trailerLote = addMeta(
+      lote.trailerLote,
+      'trailerLote',
+      cnabName,
+      Number(i),
+    );
   }
-  cnab.trailerArquivo = addMeta(cnab.trailerArquivo, 'trailerArquivo', cnabName);
+  cnab.trailerArquivo = addMeta(
+    cnab.trailerArquivo,
+    'trailerArquivo',
+    cnabName,
+  );
 }
 
 export function setCnab104Metadata(cnab: CnabFile104, cnabName?: string) {
   cnab.headerArquivo = addMeta(cnab.headerArquivo, 'headerArquivo', cnabName);
   for (const i in cnab.lotes) {
     const lote = cnab.lotes[i];
-    lote.headerLote = addMeta(lote.headerLote, 'headerLote', cnabName, Number(i));
+    lote.headerLote = addMeta(
+      lote.headerLote,
+      'headerLote',
+      cnabName,
+      Number(i),
+    );
     for (const registro of lote.registros) {
-      registro.detalheA = addMeta(registro.detalheA, 'detalheA', cnabName, Number(i));
-      registro.detalheB = addMeta(registro.detalheB, 'detalheB', cnabName, Number(i));
-      registro.detalheE = addMeta(registro.detalheE, 'detalheE', cnabName, Number(i));
+      registro.detalheA = addMeta(
+        registro.detalheA,
+        'detalheA',
+        cnabName,
+        Number(i),
+      );
+      registro.detalheB = addMeta(
+        registro.detalheB,
+        'detalheB',
+        cnabName,
+        Number(i),
+      );
+      registro.detalheE = addMeta(
+        registro.detalheE,
+        'detalheE',
+        cnabName,
+        Number(i),
+      );
     }
-    lote.trailerLote = addMeta(lote.trailerLote, 'trailerLote', cnabName, Number(i));
+    lote.trailerLote = addMeta(
+      lote.trailerLote,
+      'trailerLote',
+      cnabName,
+      Number(i),
+    );
   }
-  cnab.trailerArquivo = addMeta(cnab.trailerArquivo, 'trailerArquivo', cnabName);
+  cnab.trailerArquivo = addMeta(
+    cnab.trailerArquivo,
+    'trailerArquivo',
+    cnabName,
+  );
 }
 
 /**
@@ -57,14 +108,14 @@ export function addCnabTemplateMetadata<T>(
       registro: registroName,
       cnab: cnabName || _fields[key]._metadata?.cnab,
       registroIndex: loteNumber || _fields[key]._metadata?.registroIndex,
-    }
+    };
   }
   return fields;
 }
 
 export function getCnabFieldNameLog(field: CnabField) {
   const cnabName = field._metadata?.cnab || '';
-  const regName = ((x = field._metadata?.registro) => x ? `.${x}` : '');
-  const fieldName = ((x = field._metadata?.name) => x ? `.${x}` : '');
+  const regName = (x = field._metadata?.registro) => (x ? `.${x}` : '');
+  const fieldName = (x = field._metadata?.name) => (x ? `.${x}` : '');
   return cnabName + regName() + fieldName();
-}
+}