Update transmittal sheet to include first sign date

common/src/main/scala/hmda/query/ts/TransmittalSheetEntity.scala

@@ -26,7 +26,8 @@ case class TransmittalSheetEntity(
                                    submissionId: Option[String] = Some(""),
                                    createdAt: Option[java.sql.Timestamp] = Some(new java.sql.Timestamp(System.currentTimeMillis())),
                                    isQuarterly: Option[Boolean] = Some(false),
-                                   signDate: Option[Long] = Some(0L)
+                                   signDate: Option[Long] = Some(0L),
+                                   firstSignDate: Option[Long] = Some(0L)
                                  ) extends ColumnDataFormatter {
   def isEmpty: Boolean = lei == ""
 
@@ -35,7 +36,7 @@ case class TransmittalSheetEntity(
       s"$quarter|$name|$phone|" +
       s"$email|$street|$city|" +
       s"$state|$zipCode|$agency|" +
-      s"$totalLines|$taxId|$lei|${dateToString(signDate)}"
+      s"$totalLines|$taxId|$lei|${dateToString(signDate)}, ${dateToString(firstSignDate)}"
 
   def toPublicPSV: String =
     s"$year|$quarter|$lei|$taxId|$agency|" +
@@ -88,7 +89,7 @@ object TransmittalSheetEntity {
       s"$quarter|$name|$phone|" +
       s"$email|$street|$city|" +
       s"$state|$zipCode|$agency|" +
-      s"$totalLines|$taxId|$lei|${dateToString(signDate)}"
+      s"$totalLines|$taxId|$lei|${dateToString(signDate)}|${dateToString(firstSignDate)}"
    */
   object RegulatorParser extends PsvParsingCompanion[TransmittalSheetEntity] {
     override val psvReader: cormorant.Read[TransmittalSheetEntity] = { (a: CSV.Row) =>
@@ -108,9 +109,10 @@ object TransmittalSheetEntity {
         (rest, totalLines)      <- enforcePartialRead(readNext[Int], rest)
         (rest, taxId)           <- enforcePartialRead(readNext[String], rest)
         (rest, lei)             <- enforcePartialRead(readNext[String], rest)
-        signDateOrMore          <- readNext[String].readPartial(rest)
+        (rest, signDate)        <- enforcePartialRead(readNext[String], rest)
+        firstSignDateOrMore     <- readNext[String].readPartial(rest)
       } yield {
-        def create(signDate: String) = TransmittalSheetEntity(
+        def create(firstSignDate: String) = TransmittalSheetEntity(
           lei = lei,
           id = id,
           institutionName = institutionName,
@@ -126,12 +128,14 @@ object TransmittalSheetEntity {
           agency = agency,
           totalLines = totalLines,
           taxId = taxId,
-          signDate = dateFromString(signDate)
+          signDate = dateFromString(signDate),
+          firstSignDate = dateFromString(firstSignDate)
         )
 
-        signDateOrMore match {
-          case Left((more, signDate)) => Left(more -> create(signDate))
-          case Right(signDate)        => Right(create(signDate))
+
+        firstSignDateOrMore match {
+          case Left((more, firstSignDate)) => Left(more -> create(firstSignDate))
+          case Right(firstSignDate)        => Right(create(firstSignDate))
         }
       }
     }

common/src/main/scala/hmda/query/ts/TransmittalSheetTable.scala

@@ -25,6 +25,7 @@ class TransmittalSheetTable(tag: Tag, tableName: String) extends Table[Transmitt
   def createdAt = column[Option[Timestamp]]("created_at")
   def isQuarterly = column[Option[Boolean]]("is_quarterly")
   def signDate = column[Option[Long]]("sign_date")
+  def firstSignDate = column[Option[Long]]("sign_date")
 
   override def * =
     (
@@ -46,6 +47,7 @@ class TransmittalSheetTable(tag: Tag, tableName: String) extends Table[Transmitt
       submissionId,
       createdAt,
       isQuarterly,
-      signDate
+      signDate,
+      firstSignDate
     ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
 }

docs/spec/Agency_TS_Spec.csv

@@ -14,4 +14,5 @@ agency, Federal Agency, Numeric, HMDA Filing Instructions Guide
 totalLines, Total Number of Entries Contained in Submission, Numeric, HMDA Filing Instructions Guide
 taxId, Federal Taxpayer Identification Number, Alphanumeric, HMDA Filing Instructions Guide
 lei, Legal Entity Identifier (LEI), Alphanumeric, HMDA Filing Instructions Guide
-signDate, HMDA Filing Sign Datetime, Alphanumeric, Collected by the HMDA Platform
+signDate, HMDA Filing Sign Datetime, Alphanumeric, Collected by the HMDA Platform
+firstSignDate, HMDA Filing First Sign Datetime, Alphanumeric, Collected by the HMDA Platform

hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala

@@ -140,6 +140,10 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
         .map(l => l.submission.end)
         .runWith(Sink.lastOption)
 
+    def firstSignDateSubmissionHistory: Future[Seq[Long]] = {
+      submissionHistoryRepository.firstSignDate(submissionId)
+    }
+
     def deleteTsRow: Future[Done] =
       rawData.take(1)
         .map(s => TsCsvParser(s, fromCassandra = true))
@@ -189,16 +193,29 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
           val enforceQuarterly = submissionId.period.quarter.isDefined
           for {
             signdate <- signDate
-            insertorupdate <- repo.insert(copyTs(ts, Some(signdate.getOrElse(0L)), enforceQuarterly))
+            firstsigndate <- firstSignDateSubmissionHistory
+            insertorupdate <- {
+              println("first sign date")
+              println(firstsigndate)
+              val resolvedSignDate = Some(signdate.getOrElse(0L))
+              val resolvedFirstSignDate = {
+                if (firstsigndate.isEmpty) resolvedSignDate
+                else Some(firstsigndate.head)
+              }
+              repo.insert(copyTs(ts, resolvedSignDate, resolvedFirstSignDate, enforceQuarterly))
+            }
           } yield insertorupdate
         }
         .runWith(Sink.ignore)
 
-    def copyTs(ts: TransmittalSheetEntity, signdate: Option[Long], enforceQuarterly: Boolean): TransmittalSheetEntity =
+    def copyTs(ts: TransmittalSheetEntity, signdate: Option[Long], firstsigndate: Option[Long], enforceQuarterly: Boolean): TransmittalSheetEntity =
       if (enforceQuarterly) {
-        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate, isQuarterly = Some(true))
+        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate, firstSignDate = firstsigndate, isQuarterly = Some(true))
       } else {
-        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate)
+        val newts = ts.copy(lei = ts.lei.toUpperCase, firstSignDate = firstsigndate, signDate = signdate)
+        println("new ts")
+        println(newts)
+        newts
       }
 
     def deleteLarRows: Future[Done] =
@@ -248,6 +265,9 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
     def result =
       for {
 
+        firstSignDate <- firstSignDateSubmissionHistory
+        _ = log.info(s"First date signed $firstSignDate")
+
         _ <- deleteTsRow
         _ = if(tsDeletion)
           log.info(s"Attempt to remove data from TS table for  $submissionId  completed.")
@@ -268,7 +288,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
         _ = log.info(s"Attempt to add data to LAR table for  $submissionId  completed.")
 
 
-        dateSigned   <- signDate
+        dateSigned <- signDate
         _ = log.info(s"Date signed $dateSigned")
 
         res <- insertSubmissionHistory

hmda-analytics/src/main/scala/hmda/analytics/query/SubmissionHistoryComponent.scala

@@ -28,6 +28,18 @@ trait SubmissionHistoryComponent {
            sign_date = ${signDate}
           """
       }
+
+    def firstSignDate(submissionId: SubmissionId): Future[Seq[Long]] = {
+      val period = submissionId.period.year
+      val lei = submissionId.lei
+      val submissionIdLikeStatment = s"${lei}-${period}-%"
+      config.db.run {
+        sql"""
+        SELECT MIN(sign_date) from #${tableName}
+        WHERE submission_id LIKE $submissionIdLikeStatment
+        """.as[Long]
+      }
+    }
   }
 
 }

hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala

@@ -35,6 +35,7 @@ trait TransmittalSheetComponent {
     def createdAt       = column[Option[Timestamp]]("created_at")
     def isQuarterly     = column[Option[Boolean]]("is_quarterly")
     def signDate     = column[Option[Long]]("sign_date")
+    def firstSignDate   = column[Option[Long]]("first_sign_data")
 
     override def * =
       (
@@ -56,7 +57,8 @@ trait TransmittalSheetComponent {
         submissionId,
         createdAt,
         isQuarterly,
-        signDate
+        signDate,
+        firstSignDate
       ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
   }
 
@@ -84,6 +86,9 @@ trait TransmittalSheetComponent {
     def findByLei(lei: String): Future[Seq[TransmittalSheetEntity]] =
       db.run(table.filter(_.lei.toUpperCase === lei.toUpperCase).result)
 
+    def findByLeiAndQuarter(lei: String): Future[Seq[TransmittalSheetEntity]] =
+      db.run(table.filter(x => x.lei.toUpperCase === lei.toUpperCase && x.isQuarterly === true).result)
+
     def deleteByLei(lei: String): Future[Int] =
       db.run(table.filter(_.lei.toUpperCase === lei.toUpperCase).delete)
 

hmda-data-publisher/src/main/scala/hmda/publisher/query/component/SubmissionHistoryComponent.scala

@@ -0,0 +1,28 @@
+package hmda.publisher.query.component
+
+import hmda.publisher.helper.PGTableNameLoader
+import hmda.publisher.query.panel.InstitutionEmailEntity
+import hmda.query.DbConfiguration.dbConfig
+import hmda.query.repository.TableRepository
+import slick.basic.DatabaseConfig
+import slick.jdbc.JdbcProfile
+
+import scala.concurrent.Future
+
+trait SubmissionHistoryComponent extends PGTableNameLoader{
+
+
+  import dbConfig.profile.api._
+
+  class SubmissionHistoryTable(tag: Tag)
+    extends Table[SubmissionHistoryEntity](tag, submissionHistoryTableName) with SubmissionHistoryComponent {
+    def lei = column[String]("lei")
+    def submissionId= column[String]("submission_id")
+    def signDate = column[BigInt]("sign_date")
+
+    def * =
+      (lei, submissionId, signDate) <> (SubmissionHistoryEntity.tupled, SubmissionHistoryEntity.unapply)
+
+  }
+
+}

hmda-data-publisher/src/main/scala/hmda/publisher/query/component/TransmittalSheetTable.scala

@@ -27,6 +27,7 @@ abstract class AbstractTransmittalSheetTable[T](tag: Tag, tableName: String) ext
   def createdAt       = column[Option[Timestamp]]("created_at")
   def isQuarterly     = column[Option[Boolean]]("is_quarterly")
   def signDate        = column[Option[Long]]("sign_date")
+  def firstSignDate   = column[Option[Long]]("first_sign_date")
 
   def transmittalSheetEntityProjection =
     (
@@ -48,7 +49,8 @@ abstract class AbstractTransmittalSheetTable[T](tag: Tag, tableName: String) ext
       submissionId,
       createdAt,
       isQuarterly,
-      signDate
+      signDate,
+      firstSignDate
     ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
 }