Merge from docusealco/wip

.rubocop.yml

@@ -48,6 +48,9 @@ Style/MultipleComparison:
 Style/NumericPredicate:
   Enabled: false
 
+Style/MinMaxComparison:
+  Enabled: false
+
 Naming/PredicateMethod:
   Enabled: false
 

Dockerfile

@@ -9,7 +9,7 @@ RUN apk --no-cache add fontforge wget && \
     wget https://cdn.jsdelivr.net/gh/notofonts/notofonts.github.io/fonts/NotoSansSymbols2/hinted/ttf/NotoSansSymbols2-Regular.ttf && \
     wget https://github.com/Maxattax97/gnu-freefont/raw/master/ttf/FreeSans.ttf && \
     wget https://github.com/impallari/DancingScript/raw/master/OFL.txt && \
-    wget -O /model.onnx "https://github.com/docusealco/fields-detection/releases/download/2.0.0/model_704_int8.onnx" && \
+    wget -O /model.onnx "https://github.com/docusealco/fields-detection/releases/download/2.1.0/model_704_int8.onnx" && \
     wget -O pdfium-linux.tgz "https://github.com/docusealco/pdfium-binaries/releases/latest/download/pdfium-linux-$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/').tgz" && \
     mkdir -p /pdfium-linux && \
     tar -xzf pdfium-linux.tgz -C /pdfium-linux
@@ -53,6 +53,8 @@ WORKDIR /app
 
 RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev yaml-dev redis libheif vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf
 
+RUN addgroup -g 2000 docuseal && adduser -u 2000 -G docuseal -s /bin/sh -D -h /home/docuseal docuseal
+
 RUN echo $'.include = /etc/ssl/openssl.cnf\n\
 \n\
 [provider_sect]\n\
@@ -92,7 +94,10 @@ COPY --from=webpack /app/public/packs ./public/packs
 RUN ln -s /fonts /app/public/fonts
 RUN bundle exec bootsnap precompile -j 1 --gemfile app/ lib/
 
+RUN chown -R docuseal:docuseal /app
+
 WORKDIR /data/docuseal
+ENV HOME=/home/docuseal
 ENV WORKDIR=/data/docuseal
 
 EXPOSE 3000

app/controllers/api/submission_documents_controller.rb

@@ -5,34 +5,17 @@ module Api
     load_and_authorize_resource :submission
 
     def index
+      is_merge = params[:merge] == 'true' &&
+                 (@submission.schema_documents || @submission.template.schema_documents).size > 1
+
       documents =
         if @submission.submitters.all?(&:completed_at?)
-          last_submitter = @submission.submitters.max_by(&:completed_at)
-
-          if last_submitter.documents_attachments.blank?
-            last_submitter.documents_attachments = Submissions::EnsureResultGenerated.call(last_submitter)
-          end
-
-          last_submitter.documents_attachments
+          build_completed_documents(@submission, merge: is_merge)
         else
-          values_hash = Submissions::GeneratePreviewAttachments.build_values_hash(@submission)
-
-          if @submission.preview_documents.present? &&
-             @submission.preview_documents.all? { |s| s.metadata['values_hash'] == values_hash }
-            @submission.preview_documents
-          else
-            ApplicationRecord.no_touching do
-              @submission.preview_documents.each(&:destroy)
-            end
-
-            Submissions::GeneratePreviewAttachments.call(@submission, values_hash:)
-          end
+          build_preview_documents(@submission, merge: is_merge)
         end
 
-      ActiveRecord::Associations::Preloader.new(
-        records: documents,
-        associations: [:blob]
-      ).call
+      ActiveRecord::Associations::Preloader.new(records: documents, associations: [:blob]).call
 
       expires_at = Accounts.link_expires_at(current_account)
 
@@ -43,5 +26,50 @@ module Api
         end
       }
     end
+
+    private
+
+    def build_completed_documents(submission, merge: false)
+      last_submitter = submission.submitters.max_by(&:completed_at)
+
+      if merge
+        if submission.merged_document_attachment.blank?
+          submission.merged_document_attachment =
+            Submissions::GenerateCombinedAttachment.call(last_submitter, with_audit: false)
+        end
+
+        [submission.merged_document_attachment]
+      else
+        if last_submitter.documents_attachments.blank?
+          last_submitter.documents_attachments = Submissions::EnsureResultGenerated.call(last_submitter)
+        end
+
+        last_submitter.documents_attachments
+      end
+    end
+
+    def build_preview_documents(submission, merge: false)
+      values_hash = Submissions::GeneratePreviewAttachments.build_values_hash(submission)
+
+      if merge
+        if submission.preview_merged_document_attachment.present? &&
+           submission.preview_merged_document_attachment.metadata['values_hash'] == values_hash
+          [submission.preview_merged_document_attachment]
+        else
+          ApplicationRecord.no_touching { submission.preview_merged_document_attachment&.destroy }
+
+          Submissions::GeneratePreviewAttachments.call(submission, values_hash:, merge: true)
+        end
+      elsif submission.preview_documents.present? &&
+            submission.preview_documents.all? { |s| s.metadata['values_hash'] == values_hash }
+        submission.preview_documents
+      else
+        ApplicationRecord.no_touching do
+          submission.preview_documents.each(&:destroy)
+        end
+
+        Submissions::GeneratePreviewAttachments.call(submission, values_hash:)
+      end
+    end
   end
 end

app/controllers/application_controller.rb

@@ -17,6 +17,7 @@ class ApplicationController < ActionController::Base
 
   helper_method :button_title,
                 :current_account,
+                :true_ability,
                 :form_link_host,
                 :svg_icon
 
@@ -102,6 +103,10 @@ class ApplicationController < ActionController::Base
     current_user&.account
   end
 
+  def true_ability
+    @true_ability ||= Ability.new(true_user)
+  end
+
   def maybe_redirect_to_setup
     redirect_to setup_index_path unless User.exists?
   end

app/controllers/templates_clone_controller.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class TemplatesCloneController < ApplicationController
+  load_and_authorize_resource :template, instance_name: :base_template
+
+  def new
+    authorize!(:create, Template)
+
+    @template = Template.new(name: "#{@base_template.name} (#{I18n.t('clone')})")
+  end
+
+  def create
+    ActiveRecord::Associations::Preloader.new(
+      records: [@base_template],
+      associations: [schema_documents: :preview_images_attachments]
+    ).call
+
+    @template = Templates::Clone.call(@base_template, author: current_user,
+                                                      name: params.dig(:template, :name),
+                                                      folder_name: params[:folder_name])
+
+    authorize!(:create, @template)
+
+    if params[:account_id].present? && true_ability.authorize!(:manage, Account.find(params[:account_id]))
+      @template.account_id = params[:account_id]
+      @template.author = true_user if true_user.account_id == @template.account_id
+      @template.folder = @template.account.default_template_folder if @template.account_id != current_account.id
+    else
+      @template.account = current_account
+    end
+
+    Templates.maybe_assign_access(@template)
+
+    if @template.save
+      Templates::CloneAttachments.call(template: @template, original_template: @base_template)
+
+      SearchEntries.enqueue_reindex(@template)
+
+      WebhookUrls.enqueue_events(@template, 'template.created')
+
+      maybe_redirect_to_template(@template)
+    else
+      render turbo_stream: turbo_stream.replace(:modal, partial: 'templates_clone/form'), status: :unprocessable_content
+    end
+  end
+
+  private
+
+  def maybe_redirect_to_template(template)
+    if template.account == current_account
+      redirect_to(edit_template_path(template))
+    else
+      redirect_back(fallback_location: root_path, notice: I18n.t('template_has_been_cloned'))
+    end
+  end
+end

app/controllers/templates_controller.rb

@@ -3,8 +3,6 @@
 class TemplatesController < ApplicationController
   load_and_authorize_resource :template
 
-  before_action :load_base_template, only: %i[new create]
-
   def show
     submissions = @template.submissions.accessible_by(current_ability)
     submissions = submissions.active if @template.archived_at.blank?
@@ -26,9 +24,7 @@ class TemplatesController < ApplicationController
     redirect_to root_path
   end
 
-  def new
-    @template.name = "#{@base_template.name} (#{I18n.t('clone')})" if @base_template
-  end
+  def new; end
 
   def edit
     ActiveRecord::Associations::Preloader.new(
@@ -48,37 +44,18 @@ class TemplatesController < ApplicationController
   end
 
   def create
-    if @base_template
-      ActiveRecord::Associations::Preloader.new(
-        records: [@base_template],
-        associations: [schema_documents: :preview_images_attachments]
-      ).call
-
-      @template = Templates::Clone.call(@base_template, author: current_user,
-                                                        name: params.dig(:template, :name),
-                                                        folder_name: params[:folder_name])
-    else
-      @template.author = current_user
-      @template.folder = TemplateFolders.find_or_create_by_name(current_user, params[:folder_name])
-    end
-
-    if params[:account_id].present? && authorized_clone_account_id?(params[:account_id])
-      @template.account_id = params[:account_id]
-      @template.folder = @template.account.default_template_folder if @template.account_id != current_account.id
-    else
-      @template.account = current_account
-    end
+    @template.author = current_user
+    @template.folder = TemplateFolders.find_or_create_by_name(current_user, params[:folder_name])
+    @template.account = current_account
 
     Templates.maybe_assign_access(@template)
 
     if @template.save
-      Templates::CloneAttachments.call(template: @template, original_template: @base_template) if @base_template
-
       SearchEntries.enqueue_reindex(@template)
 
       WebhookUrls.enqueue_events(@template, 'template.created')
 
-      maybe_redirect_to_template(@template)
+      redirect_to(edit_template_path(@template))
     else
       render turbo_stream: turbo_stream.replace(:modal, template: 'templates/new'), status: :unprocessable_content
     end
@@ -132,23 +109,4 @@ class TemplatesController < ApplicationController
                     areas: [%i[x y w h cell_w attachment_uuid option_uuid page]] }]] }
     )
   end
-
-  def authorized_clone_account_id?(account_id)
-    true_user.account_id.to_s == account_id.to_s ||
-      true_user.account.linked_accounts.accessible_by(current_ability).exists?(id: account_id)
-  end
-
-  def maybe_redirect_to_template(template)
-    if template.account == current_account
-      redirect_to(edit_template_path(@template))
-    else
-      redirect_back(fallback_location: root_path, notice: I18n.t('template_has_been_cloned'))
-    end
-  end
-
-  def load_base_template
-    return if params[:base_template_id].blank?
-
-    @base_template = Template.accessible_by(current_ability).find_by(id: params[:base_template_id])
-  end
 end

app/javascript/submission_form/area.vue

@@ -461,7 +461,7 @@ export default {
         phone: IconPhoneCheck,
         payment: IconCreditCard,
         verification: IconId,
-        kba: IconUserScan,
+        kba: IconUserScan
       }
     },
     image () {

app/javascript/submission_form/kba_step.vue

@@ -10,9 +10,15 @@
     <template v-else>{{ field.name || 'Knowledge Based Authentication' }}</template>
     <span
       v-if="questions"
-      class="float-right text-base font-normal text-neutral-500 mt-1 whitespace-nowrap"
+      class="float-right font-normal text-neutral-500 mt-1 whitespace-nowrap text-right text-sm"
     >
-      Question {{ currentQuestionIndex + 1 }} / {{ questions.length }}
+      <span>Question {{ currentQuestionIndex + 1 }} / {{ questions.length }}</span>
+      <span
+        v-if="timeLeftSeconds !== null"
+        class="block"
+      >
+        Time left: {{ formattedTimeLeft }}
+      </span>
     </span>
   </label>
   <div
@@ -54,7 +60,9 @@
   <div v-else-if="questions && !error">
     <form @submit.prevent="nextQuestion">
       <div class="mb-6 px-1">
-        <p class="font-semibold mb-4 text-lg">{{ currentQuestion.prompt }}</p>
+        <p class="font-semibold mb-4 text-lg">
+          {{ currentQuestion.prompt }}
+        </p>
         <div class="space-y-3.5 mx-auto">
           <div
             v-for="(answer, index) in currentQuestion.answers"
@@ -316,6 +324,8 @@ export default {
       reference: null,
       answers: {},
       error: null,
+      timeLeftSeconds: null,
+      countdownIntervalId: null,
       form: {
         fn: '',
         ln: '',
@@ -337,6 +347,9 @@ export default {
     isRequiredFieldEmpty () {
       return this.emptyValueRequiredStep && this.emptyValueRequiredStep[0] !== this.field
     },
+    kbaTimeLimitSeconds () {
+      return 90
+    },
     states () {
       return [
         { code: 'AL', name: 'Alabama' },
@@ -394,9 +407,53 @@ export default {
     },
     isLastQuestion () {
       return this.questions && this.currentQuestionIndex === this.questions.length - 1
+    },
+    formattedTimeLeft () {
+      if (this.timeLeftSeconds === null) return ''
+
+      const minutes = Math.floor(this.timeLeftSeconds / 60)
+      const seconds = this.timeLeftSeconds % 60
+
+      return `${minutes}:${String(seconds).padStart(2, '0')}`
     }
   },
+  beforeUnmount () {
+    this.clearCountdown()
+  },
   methods: {
+    clearCountdown () {
+      if (this.countdownIntervalId) {
+        clearInterval(this.countdownIntervalId)
+        this.countdownIntervalId = null
+      }
+
+      this.timeLeftSeconds = null
+    },
+    startCountdown () {
+      this.clearCountdown()
+
+      this.timeLeftSeconds = this.kbaTimeLimitSeconds
+
+      this.countdownIntervalId = setInterval(() => {
+        if (this.timeLeftSeconds === null) return
+
+        this.timeLeftSeconds -= 1
+
+        if (this.timeLeftSeconds <= 0) {
+          this.handleTimeout()
+        }
+      }, 1000)
+    },
+    handleTimeout () {
+      this.clearCountdown()
+
+      this.questions = null
+      this.token = null
+      this.reference = null
+      this.answers = {}
+      this.currentQuestionIndex = 0
+      this.error = `Knowledge Based Authentication timed out. You only have ${this.kbaTimeLimitSeconds} seconds to complete the verification. Please retry.`
+    },
     nextQuestion () {
       if (this.isLastQuestion) {
         this.$emit('submit')
@@ -405,6 +462,8 @@ export default {
       }
     },
     restartKba () {
+      this.clearCountdown()
+
       this.questions = null
       this.token = null
       this.reference = null
@@ -415,6 +474,7 @@ export default {
     async startKba () {
       this.isLoading = true
       this.error = null
+      this.clearCountdown()
 
       try {
         const payload = { ...this.form, submitter_slug: this.submitterSlug }
@@ -457,6 +517,8 @@ export default {
           this.questions.forEach(q => {
             this.answers[q.id] = null
           })
+
+          this.startCountdown()
         } else {
           throw new Error('Invalid KBA response')
         }
@@ -467,6 +529,8 @@ export default {
       }
     },
     async submit () {
+      this.clearCountdown()
+
       this.isSubmitting = true
       this.error = null
 

app/javascript/template_builder/builder.vue

@@ -1116,14 +1116,17 @@ export default {
       const sortArea = (aArea, bArea) => {
         if (aArea.attachment_uuid === bArea.attachment_uuid) {
           if (aArea.page === bArea.page) {
-            if (Math.abs(aArea.y - bArea.y) < 0.01) {
+            const aY = aArea.y + aArea.h
+            const bY = bArea.y + bArea.h
+
+            if (Math.abs(aY - bY) < 0.01 || (aArea.h < bArea.h ? (aArea.y >= bArea.y && aY <= bY) : (bArea.y >= aArea.y && bY <= aY))) {
               if (aArea.x === bArea.x) {
                 return 0
               } else {
                 return aArea.x - bArea.x
               }
             } else {
-              return aArea.y - bArea.y
+              return (aArea.y + aArea.h) - (bArea.y + bArea.h)
             }
           } else {
             return aArea.page - bArea.page
@@ -1173,6 +1176,72 @@ export default {
         this.save()
       }
     },
+    findFieldInsertIndex (field) {
+      if (!field.areas?.length) return -1
+
+      const area = field.areas[0]
+
+      const attachmentUuidsIndex = this.template.schema.reduce((acc, e, index) => {
+        acc[e.attachment_uuid] = index
+
+        return acc
+      }, {})
+
+      const compareAreas = (a, b) => {
+        const aAttIdx = attachmentUuidsIndex[a.attachment_uuid]
+        const bAttIdx = attachmentUuidsIndex[b.attachment_uuid]
+
+        if (aAttIdx !== bAttIdx) return aAttIdx - bAttIdx
+        if (a.page !== b.page) return a.page - b.page
+
+        const aY = a.y + a.h
+        const bY = b.y + b.h
+
+        if (Math.abs(aY - bY) < 0.01) return a.x - b.x
+        if (a.h < b.h ? a.y >= b.y && aY <= bY : b.y >= a.y && bY <= aY) return a.x - b.x
+
+        return aY - bY
+      }
+
+      let closestBeforeIndex = -1
+      let closestBeforeArea = null
+      let closestAfterIndex = -1
+      let closestAfterArea = null
+
+      this.template.fields.forEach((f, index) => {
+        if (f.submitter_uuid === field.submitter_uuid) {
+          (f.areas || []).forEach((a) => {
+            const cmp = compareAreas(a, area)
+
+            if (cmp < 0) {
+              if (!closestBeforeArea || (compareAreas(a, closestBeforeArea) > 0 && closestBeforeIndex < index)) {
+                closestBeforeIndex = index
+                closestBeforeArea = a
+              }
+            } else {
+              if (!closestAfterArea || (compareAreas(a, closestAfterArea) < 0 && closestAfterIndex > index)) {
+                closestAfterIndex = index
+                closestAfterArea = a
+              }
+            }
+          })
+        }
+      })
+
+      if (closestBeforeIndex !== -1) return closestBeforeIndex + 1
+      if (closestAfterIndex !== -1) return closestAfterIndex
+
+      return -1
+    },
+    insertField (field) {
+      const insertIndex = this.findFieldInsertIndex(field)
+
+      if (insertIndex !== -1) {
+        this.template.fields.splice(insertIndex, 0, field)
+      } else {
+        this.template.fields.push(field)
+      }
+    },
     closeDropdown () {
       document.activeElement.blur()
     },
@@ -1226,7 +1295,7 @@ export default {
         field.preferences.with_signature_id = this.withSignatureId
       }
 
-      this.template.fields.push(field)
+      this.insertField(field)
 
       this.save()
     },
@@ -1455,11 +1524,13 @@ export default {
 
           field.areas.push(area)
         } else {
-          this.template.fields.push({
+          const newField = {
             ...JSON.parse(JSON.stringify(field)),
             uuid: v4(),
             areas: [area]
-          })
+          }
+
+          this.insertField(newField)
         }
 
         this.selectedAreaRef.value = area
@@ -1482,7 +1553,7 @@ export default {
       const documentRef = this.documentRefs.find((e) => e.document.uuid === area.attachment_uuid)
       const pageMask = documentRef.pageRefs[area.page].$refs.mask
 
-      if (type === 'checkbox') {
+      if (type === 'checkbox' || type === 'radio' || type === 'multiple') {
         area.w = pageMask.clientWidth / 30 / pageMask.clientWidth
         area.h = (pageMask.clientWidth / 30 / pageMask.clientWidth) * (pageMask.clientWidth / pageMask.clientHeight)
       } else if (type === 'image') {
@@ -1544,7 +1615,7 @@ export default {
         }
 
         if (this.template.fields.indexOf(this.drawField) === -1) {
-          this.template.fields.push(this.drawField)
+          this.insertField(this.drawField)
         }
 
         this.drawField = null
@@ -1572,8 +1643,8 @@ export default {
           const previousArea = previousField?.areas?.[previousField.areas.length - 1]
 
           if (previousArea || area.w) {
-            const areaW = previousArea?.w || (30 / pageMask.clientWidth)
-            const areaH = previousArea?.h || (30 / pageMask.clientHeight)
+            const areaW = previousArea?.w || area.w || (30 / pageMask.clientWidth)
+            const areaH = previousArea?.h || area.h || (30 / pageMask.clientHeight)
 
             if ((pageMask.clientWidth * area.w) < 5) {
               area.x = area.x - (areaW / 2)
@@ -1683,7 +1754,7 @@ export default {
       this.selectedAreaRef.value = fieldArea
 
       if (this.template.fields.indexOf(field) === -1) {
-        this.template.fields.push(field)
+        this.insertField(field)
       }
 
       this.save()
@@ -1713,7 +1784,7 @@ export default {
       } else if (previousField?.areas?.length) {
         baseArea = previousField.areas[previousField.areas.length - 1]
       } else {
-        if (['checkbox'].includes(fieldType)) {
+        if (['checkbox', 'radio', 'multiple'].includes(fieldType)) {
           baseArea = {
             w: area.maskW / 30 / area.maskW,
             h: area.maskW / 30 / area.maskW * (area.maskW / area.maskH)
@@ -1845,7 +1916,7 @@ export default {
           attachment.metadata.pdf.fields.forEach((field) => {
             field.submitter_uuid = this.selectedSubmitter.uuid
 
-            this.template.fields.push(field)
+            this.insertField(field)
           })
         }
       })

app/javascript/template_builder/field_type.vue

@@ -160,7 +160,7 @@ export default {
         payment: IconCreditCard,
         phone: IconPhoneCheck,
         verification: IconId,
-        kba: IconUserScan,
+        kba: IconUserScan
       }
     },
     skipTypes () {

app/models/submission.rb

@@ -63,6 +63,8 @@ class Submission < ApplicationRecord
 
   has_one_attached :audit_trail
   has_one_attached :combined_document
+  has_one_attached :merged_document
+  has_one_attached :preview_merged_document
 
   has_many_attached :preview_documents
   has_many_attached :documents

app/views/accounts/show.html.erb

@@ -235,7 +235,7 @@
             <% end %>
           <% end %>
         <% end %>
-        <% if can?(:manage, :personalization_advanced) %>
+        <% if !Docuseal.multitenant? || can?(:manage, :personalization_advanced) %>
           <% account_config = AccountConfig.find_or_initialize_by(account: current_account, key: AccountConfig::WITH_FILE_LINKS_KEY) %>
           <% if can?(:manage, account_config) %>
             <%= form_for account_config, url: account_configs_path, method: :post do |f| %>

app/views/templates/_file_form.html.erb

@@ -1,12 +1,4 @@
 <%= form_for @template, data: { turbo_frame: :_top }, html: { autocomplete: :off } do |f| %>
-  <% if @base_template %>
-    <%= hidden_field_tag :base_template_id, @base_template.id %>
-  <% end %>
-  <% if @base_template && (can?(:manage, :tenants) || true_user != current_user) && true_user.account.linked_accounts.active.accessible_by(current_ability).exists? %>
-    <div class="form-control -mb-2 mt-2">
-      <%= select_tag :account_id, options_for_select([true_user.account, *true_user.account.linked_accounts.active.accessible_by(current_ability)].uniq.map { |e| [e.name, e.id] }, current_account.id), required: true, class: 'base-select' %>
-    </div>
-  <% end %>
   <div class="form-control mt-4">
     <%= f.text_field :name, required: true, placeholder: t('document_name'), class: 'base-input', dir: 'auto' %>
   </div>
@@ -16,7 +8,7 @@
     </label>
     <folder-autocomplete class="flex justify-between w-full">
       <set-value data-on="blur" data-value="<%= TemplateFolder::DEFAULT_NAME %>" data-empty-only="true" class="peer w-full whitespace-nowrap">
-        <input id="folder_name" placeholder="<%= t('folder_name') %>" type="text" class="w-full outline-none border-transparent focus:border-transparent focus:ring-0 bg-base-100 px-1" name="folder_name" value="<%= params[:folder_name].presence || @base_template&.folder&.full_name || TemplateFolder::DEFAULT_NAME %>" autocomplete="off">
+        <input id="folder_name" placeholder="<%= t('folder_name') %>" type="text" class="w-full outline-none border-transparent focus:border-transparent focus:ring-0 bg-base-100 px-1" name="folder_name" value="<%= params[:folder_name].presence || TemplateFolder::DEFAULT_NAME %>" autocomplete="off">
       </set-value>
       <set-value data-on="click" data-value="" data-input-id="folder_name" class="peer-focus-within:hidden whitespace-nowrap">
         <label for="folder_name" data-clear-on-focus="true" class="shrink-0 link mr-1.5 cursor-pointer">
@@ -26,6 +18,6 @@
     </folder-autocomplete>
   </div>
   <div class="form-control">
-    <%= f.button button_title(title: @base_template ? t('submit') : t('create'), disabled_with: t('creating')), class: 'base-button' %>
+    <%= f.button button_title(title: t('create'), disabled_with: t('creating')), class: 'base-button' %>
   </div>
 <% end %>

app/views/templates/_template.html.erb

@@ -53,7 +53,7 @@
       <% end %>
       <% if can?(:create, template) %>
         <span class="tooltip tooltip-left" data-tip="<%= t('clone') %>">
-          <a href="<%= new_template_path(base_template_id: template.id) %>" data-turbo-frame="modal" class="btn btn-xs hover:btn-outline bg-base-200 btn-circle">
+          <a href="<%= new_template_clone_path(template) %>" data-turbo-frame="modal" class="btn btn-xs hover:btn-outline bg-base-200 btn-circle">
             <%= svg_icon('copy', class: 'w-4 h-4') %>
           </a>
         </span>

app/views/templates/_title.html.erb

@@ -63,7 +63,7 @@
       <%= button_to button_title(title: t('archive'), disabled_with: t('archiving'), title_class: 'inline', icon: svg_icon('archive', class: 'w-6 h-6')), template_path(template), class: 'btn btn-outline btn-sm w-full', form_class: 'flex-1', method: :delete, data: { turbo_confirm: t('are_you_sure_') } %>
     <% end %>
     <% if can?(:create, current_account.templates.new(author: current_user)) %>
-      <%= link_to new_template_path(base_template_id: template.id), class: 'btn btn-outline btn-sm flex-1', data: { turbo_frame: :modal } do %>
+      <%= link_to new_template_clone_path(template), class: 'btn btn-outline btn-sm flex-1', data: { turbo_frame: :modal } do %>
         <span class="flex items-center justify-center space-x-2">
           <%= svg_icon('copy', class: 'w-6 h-6') %>
           <span class="inline"><%= t('clone') %></span>

app/views/templates_clone/_form.html.erb

@@ -0,0 +1,29 @@
+<%= form_for @template, url: template_clone_index_path(@base_template), data: { turbo_frame: :_top }, html: { autocomplete: :off } do |f| %>
+  <% accounts = Account.accessible_by(true_ability).where.not(id: true_user.account.testing_accounts).where(User.where(User.arel_table[:account_id].eq(Account.arel_table[:id])).arel.exists).active %>
+  <% if (can?(:manage, :tenants) || true_user != current_user) && accounts.where.not(id: current_account.id).exists? %>
+    <div class="form-control -mb-2 mt-2">
+      <%= select_tag :account_id, options_for_select([current_account, *accounts.order(:name)].uniq.map { |e| [e.name, e.id] }, current_account.id), required: true, class: 'base-select' %>
+    </div>
+  <% end %>
+  <div class="form-control mt-4">
+    <%= f.text_field :name, required: true, placeholder: t('document_name'), class: 'base-input', dir: 'auto' %>
+  </div>
+  <div class="mt-3 mb-4 flex items-center justify-between">
+    <label for="folder_name" class="cursor-pointer">
+      <%= svg_icon('folder', class: 'w-6 h-6') %>
+    </label>
+    <folder-autocomplete class="flex justify-between w-full">
+      <set-value data-on="blur" data-value="<%= TemplateFolder::DEFAULT_NAME %>" data-empty-only="true" class="peer w-full whitespace-nowrap">
+        <input id="folder_name" placeholder="<%= t('folder_name') %>" type="text" class="w-full outline-none border-transparent focus:border-transparent focus:ring-0 bg-base-100 px-1" name="folder_name" value="<%= params[:folder_name].presence || @base_template&.folder&.full_name || TemplateFolder::DEFAULT_NAME %>" autocomplete="off">
+      </set-value>
+      <set-value data-on="click" data-value="" data-input-id="folder_name" class="peer-focus-within:hidden whitespace-nowrap">
+        <label for="folder_name" data-clear-on-focus="true" class="shrink-0 link mr-1.5 cursor-pointer">
+          <%= t('change_folder') %>
+        </label>
+      </set-value>
+    </folder-autocomplete>
+  </div>
+  <div class="form-control">
+    <%= f.button button_title(title: t('submit'), disabled_with: t('creating')), class: 'base-button' %>
+  </div>
+<% end %>

app/views/templates_clone/new.html.erb

@@ -0,0 +1,3 @@
+<%= render 'shared/turbo_modal', title: t('clone_template') do %>
+  <%= render 'templates_clone/form' %>
+<% end %>

config/dotenv.rb

@@ -1,6 +1,20 @@
 # frozen_string_literal: true
 
 if ENV['RAILS_ENV'] == 'production'
+  if Process.uid.zero?
+    begin
+      workdir = ENV.fetch('WORKDIR', '.')
+
+      if File.exist?(workdir) && File.stat(workdir).uid != 2000
+        puts 'Changing the owner of the docuseal directory...' unless Dir.empty?(workdir)
+
+        FileUtils.chown_R(2000, 2000, workdir)
+      end
+    rescue StandardError
+      puts 'Unable to change docuseal directory owner'
+    end
+  end
+
   if !ENV['AWS_SECRET_MANAGER_ID'].to_s.empty?
     require 'aws-sdk-secretsmanager'
 
@@ -30,12 +44,30 @@ if ENV['RAILS_ENV'] == 'production'
       File.write(dotenv_path, default_env)
     end
 
+    if Process.uid.zero?
+      begin
+        File.chown(0, 0, dotenv_path)
+        File.chmod(0o600, dotenv_path)
+      rescue StandardError
+        puts 'Unable to set dotenv mod'
+      end
+    end
+
     database_url = ENV.fetch('DATABASE_URL', nil)
 
     Dotenv.load(dotenv_path)
 
     ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil)
   end
+
+  unless Process.uid == 2000
+    begin
+      Process::Sys.setgid(2000)
+      Process::Sys.setuid(2000)
+    rescue StandardError
+      puts 'Unable to run as 2000:2000'
+    end
+  end
 end
 
 if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_')

config/routes.rb

@@ -92,6 +92,7 @@ Rails.application.routes.draw do
   resources :templates, only: %i[index], controller: 'templates_dashboard'
   resources :submissions_filters, only: %i[show], param: 'name'
   resources :templates, only: %i[new create edit update show destroy] do
+    resources :clone, only: %i[new create], controller: 'templates_clone'
     resource :debug, only: %i[show], controller: 'templates_debug' if Rails.env.development?
     resources :documents, only: %i[index create], controller: 'template_documents'
     resources :clone_and_replace, only: %i[create], controller: 'templates_clone_and_replace'

lib/submissions/generate_combined_attachment.rb

@@ -4,8 +4,8 @@ module Submissions
   module GenerateCombinedAttachment
     module_function
 
-    def call(submitter)
-      pdf = build_combined_pdf(submitter)
+    def call(submitter, with_audit: true)
+      pdf = build_combined_pdf(submitter, with_audit:)
 
       submission = submitter.submission
       account = submission.account
@@ -39,7 +39,7 @@ module Submissions
         blob: ActiveStorage::Blob.create_and_upload!(
           io: io.tap(&:rewind), filename: "#{submission.name || submission.template.name}.pdf"
         ),
-        name: 'combined_document',
+        name: with_audit ? 'combined_document' : 'merged_document',
         record: submission
       )
     end
@@ -58,14 +58,16 @@ module Submissions
       pdf.sign(io, write_options: { validate: false }, **sign_params)
     end
 
-    def build_combined_pdf(submitter)
+    def build_combined_pdf(submitter, with_audit:)
       pdfs_index = Submissions::GenerateResultAttachments.generate_pdfs(submitter)
 
-      audit_trail = I18n.with_locale(submitter.account.locale) do
-        Submissions::GenerateAuditTrail.build_audit_trail(submitter.submission)
-      end
+      if with_audit
+        audit_trail = I18n.with_locale(submitter.account.locale) do
+          Submissions::GenerateAuditTrail.build_audit_trail(submitter.submission)
+        end
 
-      audit_trail.dispatch_message(:complete_objects)
+        audit_trail.dispatch_message(:complete_objects)
+      end
 
       result = HexaPDF::Document.new
 
@@ -79,7 +81,7 @@ module Submissions
         pdf.pages.each { |page| result.pages << result.import(page) }
       end
 
-      audit_trail.pages.each { |page| result.pages << result.import(page) }
+      audit_trail&.pages&.each { |page| result.pages << result.import(page) }
 
       result
     end

lib/submissions/generate_preview_attachments.rb

@@ -5,7 +5,7 @@ module Submissions
     module_function
 
     # rubocop:disable Metrics
-    def call(submission, values_hash: nil, submitter: nil)
+    def call(submission, values_hash: nil, submitter: nil, merge: false)
       values_hash ||= if submitter
                         build_submitter_values_hash(submitter)
                       else
@@ -42,48 +42,74 @@ module Submissions
 
       template = submission.template
 
-      image_pdfs = []
-      original_documents = submission.schema_documents.preload(:blob)
+      if merge
+        result = HexaPDF::Document.new
 
-      result_attachments =
-        (submission.template_schema || template.schema).filter_map do |item|
+        (submission.template_schema || template.schema).each do |item|
           pdf = pdfs_index[item['attachment_uuid']]
 
-          next if pdf.nil?
+          next unless pdf
 
-          if original_documents.find { |a| a.uuid == item['attachment_uuid'] }.image?
-            pdf = GenerateResultAttachments.normalize_image_pdf(pdf)
+          pdf.dispatch_message(:complete_objects)
 
-            image_pdfs << pdf
-          end
-
-          build_pdf_attachment(pdf:, submission:, submitter:,
-                               uuid: item['attachment_uuid'],
-                               values_hash:,
-                               name: item['name'])
+          pdf.pages.each { |page| result.pages << result.import(page) }
         end
 
-      return ApplicationRecord.no_touching { result_attachments.map { |e| e.tap(&:save!) } } if image_pdfs.size < 2
-
-      images_pdf =
-        image_pdfs.each_with_object(HexaPDF::Document.new) do |pdf, doc|
-          pdf.pages.each { |page| doc.pages << doc.import(page) }
-        end
-
-      images_pdf = GenerateResultAttachments.normalize_image_pdf(images_pdf)
-
-      images_pdf_attachment =
-        build_pdf_attachment(
-          pdf: images_pdf,
+        attachment = build_pdf_attachment(
+          pdf: result,
           submission:,
-          submitter:,
-          uuid: GenerateResultAttachments.images_pdf_uuid(original_documents.select(&:image?)),
           values_hash:,
-          name: submission.name || template.name
+          name: 'preview_merged_document',
+          filename: "#{submission.name || template.name}.pdf"
         )
 
-      ApplicationRecord.no_touching do
-        (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) }
+        ApplicationRecord.no_touching { attachment.save! }
+
+        [attachment]
+      else
+        image_pdfs = []
+        original_documents = submission.schema_documents.preload(:blob)
+
+        result_attachments =
+          (submission.template_schema || template.schema).filter_map do |item|
+            pdf = pdfs_index[item['attachment_uuid']]
+
+            next if pdf.nil?
+
+            if original_documents.find { |a| a.uuid == item['attachment_uuid'] }.image?
+              pdf = GenerateResultAttachments.normalize_image_pdf(pdf)
+
+              image_pdfs << pdf
+            end
+
+            build_pdf_attachment(pdf:, submission:, submitter:,
+                                 uuid: item['attachment_uuid'],
+                                 values_hash:,
+                                 filename: "#{item['name']}.pdf")
+          end
+
+        return ApplicationRecord.no_touching { result_attachments.map { |e| e.tap(&:save!) } } if image_pdfs.size < 2
+
+        images_pdf =
+          image_pdfs.each_with_object(HexaPDF::Document.new) do |pdf, doc|
+            pdf.pages.each { |page| doc.pages << doc.import(page) }
+          end
+
+        images_pdf = GenerateResultAttachments.normalize_image_pdf(images_pdf)
+
+        images_pdf_attachment =
+          build_pdf_attachment(
+            pdf: images_pdf,
+            submission:,
+            submitter:,
+            uuid: GenerateResultAttachments.images_pdf_uuid(original_documents.select(&:image?)),
+            values_hash:,
+            filename: "#{submission.name || template.name}.pdf"
+          )
+
+        ApplicationRecord.no_touching do
+          (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) }
+        end
       end
     end
 
@@ -102,7 +128,8 @@ module Submissions
       )
     end
 
-    def build_pdf_attachment(pdf:, submission:, submitter:, uuid:, name:, values_hash:)
+    def build_pdf_attachment(pdf:, submission:, filename:, values_hash:, submitter: nil, uuid: nil,
+                             name: 'preview_documents')
       io = StringIO.new
 
       begin
@@ -114,13 +141,13 @@ module Submissions
       end
 
       ActiveStorage::Attachment.new(
-        blob: ActiveStorage::Blob.create_and_upload!(io: io.tap(&:rewind), filename: "#{name}.pdf"),
+        blob: ActiveStorage::Blob.create_and_upload!(io: io.tap(&:rewind), filename:),
         io_data: io.string,
         metadata: { original_uuid: uuid,
                     values_hash:,
                     analyzed: true,
-                    sha256: Base64.urlsafe_encode64(Digest::SHA256.digest(io.string)) },
-        name: 'preview_documents',
+                    sha256: Base64.urlsafe_encode64(Digest::SHA256.digest(io.string)) }.compact,
+        name: name,
         record: submitter || submission
       )
     end

lib/submissions/normalize_param_utils.rb

@@ -33,14 +33,18 @@ module Submissions
       return submitter_params if default_values.blank?
 
       values, new_attachments, new_fields =
-        Submitters::NormalizeValues.call(template,
-                                         default_values,
-                                         submitter_name: submitter_params[:role] ||
-                                                         template.submitters.dig(index, 'name'),
-                                         role_names: submitter_params[:roles],
-                                         for_submitter:,
-                                         add_fields:,
-                                         throw_errors: !with_values)
+        Submitters::NormalizeValues.call(
+          template,
+          default_values,
+          submitter_name: submitter_params[:role] ||
+                          (submitter_params[:uuid] &&
+                           template.submitters.find { |s| s['uuid'] == submitter_params[:uuid] }&.dig('name')) ||
+                          template.submitters.dig(index, 'name'),
+          role_names: submitter_params[:roles],
+          for_submitter:,
+          add_fields:,
+          throw_errors: !with_values
+        )
 
       submitter_params[:values] = values
 

lib/templates/detect_fields.rb

@@ -60,24 +60,24 @@ module Templates
 
     # rubocop:disable Metrics, Style
     def call(io, attachment: nil, confidence: 0.3, temperature: 1, inference: Templates::ImageToFields, nms: 0.1,
-             split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, &)
+             nmm: 0.5, split_page: false, aspect_ratio: true, padding: 20, regexp_type: true, &)
       fields, head_node =
         if attachment&.image?
-          process_image_attachment(io, attachment:, confidence:, nms:, split_page:, inference:,
+          process_image_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
                                        temperature:, aspect_ratio:, padding:, &)
         else
-          process_pdf_attachment(io, attachment:, confidence:, nms:, split_page:, inference:,
+          process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, split_page:, inference:,
                                      temperature:, aspect_ratio:, regexp_type:, padding:, &)
         end
 
       [fields, head_node]
     end
 
-    def process_image_attachment(io, attachment:, confidence:, nms:, temperature:, inference:,
+    def process_image_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
                                  split_page: false, aspect_ratio: false, padding: nil)
       image = Vips::Image.new_from_buffer(io.read, '')
 
-      fields = inference.call(image, confidence:, nms:, split_page:,
+      fields = inference.call(image, confidence:, nms:, nmm:, split_page:,
                                      temperature:, aspect_ratio:, padding:)
 
       fields = sort_fields(fields, y_threshold: 10.0 / image.height)
@@ -104,7 +104,7 @@ module Templates
       [fields, nil]
     end
 
-    def process_pdf_attachment(io, attachment:, confidence:, nms:, temperature:, inference:,
+    def process_pdf_attachment(io, attachment:, confidence:, nms:, nmm:, temperature:, inference:,
                                split_page: false, aspect_ratio: false, padding: nil, regexp_type: false)
       doc = Pdfium::Document.open_bytes(io.read)
 
@@ -121,7 +121,7 @@ module Templates
 
         image = Vips::Image.new_from_memory(data, width, height, 4, :uchar)
 
-        fields = inference.call(image, confidence: confidence / 3.0, nms:, split_page:,
+        fields = inference.call(image, confidence: confidence / 3.0, nms:, nmm:, split_page:,
                                        temperature:, aspect_ratio:, padding:)
 
         text_fields = extract_text_fields_from_page(page)

lib/templates/image_to_fields.rb

@@ -26,7 +26,7 @@ module Templates
     CPU_THREADS = Etc.nprocessors
 
     # rubocop:disable Metrics
-    def call(image, confidence: 0.3, nms: 0.1, temperature: 1,
+    def call(image, confidence: 0.3, nms: 0.1, nmm: 0.9, temperature: 1,
              split_page: false, aspect_ratio: true, padding: nil, resolution: self.resolution)
       image = image.extract_band(0, n: 3) if image.bands > 3
 
@@ -68,7 +68,7 @@ module Templates
         detections = postprocess_outputs(boxes, logits, transform_info, confidence:, temperature:, resolution:)
       end
 
-      detections = apply_nms(detections, nms)
+      detections = apply_nms_nmm(detections, nms_threshold: nms, nmm_threshold: nmm)
 
       build_fields_from_detections(detections, image)
     end
@@ -245,7 +245,7 @@ module Templates
 
       left, top, trim_width, trim_height = image.find_trim(threshold: 10, background: [255, 255, 255])
 
-      trim_width = [trim_width, image.width * 0.7].max
+      trim_width = [trim_width, image.width - (left * 1.9)].max
 
       padded_left = [left - padding, 0].max
       padded_top = [top - padding, 0].max
@@ -297,7 +297,10 @@ module Templates
       [img_array.reshape(1, 3, resolution, resolution), transform_info]
     end
 
-    def nms(boxes, scores, iou_threshold = 0.5)
+    def nms(detections, iou_threshold = 0.5)
+      boxes = detections[:xyxy]
+      scores = detections[:confidence]
+
       return Numo::Int32[] if boxes.shape[0].zero?
 
       x1 = boxes[true, 0]
@@ -333,7 +336,73 @@ module Templates
         order = order[inds + 1]
       end
 
-      Numo::Int32.cast(keep)
+      {
+        xyxy: detections[:xyxy][keep, true],
+        confidence: detections[:confidence][keep],
+        class_id: detections[:class_id][keep]
+      }
+    end
+
+    def nmm(detections, overlap_threshold = 0.9, confidence: 0.3)
+      boxes = detections[:xyxy]
+      scores = detections[:confidence]
+      classes = detections[:class_id]
+
+      return detections if boxes.shape[0].zero?
+
+      x1 = boxes[true, 0]
+      y1 = boxes[true, 1]
+      x2 = boxes[true, 2]
+      y2 = boxes[true, 3]
+
+      areas = (x2 - x1) * (y2 - y1)
+      order = areas.sort_index.reverse
+
+      keep = []
+
+      while order.size.positive?
+        i = order[0]
+        keep << i
+        break if order.size == 1
+
+        xx1 = Numo::SFloat.maximum(x1[i], x1[order[1..]])
+        yy1 = Numo::SFloat.maximum(y1[i], y1[order[1..]])
+        xx2 = Numo::SFloat.minimum(x2[i], x2[order[1..]])
+        yy2 = Numo::SFloat.minimum(y2[i], y2[order[1..]])
+
+        w = Numo::SFloat.maximum(0.0, xx2 - xx1)
+        h = Numo::SFloat.maximum(0.0, yy2 - yy1)
+
+        intersection = w * h
+
+        overlap = intersection / areas[order[1..]]
+
+        merge_mask = scores[i] > confidence ? (overlap.gt(overlap_threshold) & classes[order[1..]].eq(classes[i])) : nil
+
+        if merge_mask && (merge_inds = merge_mask.where).size.positive?
+          candidates = order[merge_inds + 1]
+
+          scores[i] = [scores[i], scores[candidates].max].max
+
+          x1[i] = [x1[i], x1[candidates].min].min
+          y1[i] = [y1[i], y1[candidates].min].min
+          x2[i] = [x2[i], x2[candidates].max].max
+          y2[i] = [y2[i], y2[candidates].max].max
+        end
+
+        if merge_mask
+          inds = (~merge_mask).where
+          order = order[inds + 1]
+        else
+          order = order[1..]
+        end
+      end
+
+      {
+        xyxy: detections[:xyxy][keep, true],
+        confidence: detections[:confidence][keep],
+        class_id: detections[:class_id][keep]
+      }
     end
 
     def postprocess_outputs(boxes, logits, transform_info, detections = nil, confidence: 0.3, temperature: 1,
@@ -429,25 +498,21 @@ module Templates
       end
     end
 
-    def apply_nms(detections, threshold = 0.5)
+    def apply_nms_nmm(detections, nms_threshold: 0.5, nmm_threshold: 0.7, confidence: 0.3)
       return detections if detections[:xyxy].shape[0].zero?
 
-      keep_indices = nms(detections[:xyxy], detections[:confidence], threshold)
+      nms_result = nms(detections, nms_threshold)
 
-      {
-        xyxy: detections[:xyxy][keep_indices, true],
-        confidence: detections[:confidence][keep_indices],
-        class_id: detections[:class_id][keep_indices]
-      }
+      nmm(nms_result, nmm_threshold, confidence:)
     end
 
     def model
       @model ||= OnnxRuntime::Model.new(
         MODEL_PATH.to_s,
-        inter_op_num_threads: CPU_THREADS,
+        inter_op_num_threads: 1,
         intra_op_num_threads: CPU_THREADS,
         enable_mem_pattern: false,
-        enable_cpu_mem_arena: false,
+        enable_cpu_mem_arena: Docuseal.multitenant? || Rails.env.development?,
         providers: ['CPUExecutionProvider']
       )
     end