-- 
-- Apply upgrade patches to survey definitions database
--

-- Version 23.07
alter table forward add column p_id integer;
alter table forward add column periodic_time time;
alter table forward add column periodic_period text;
alter table forward add column periodic_day_of_week integer;
alter table forward add column periodic_day_of_month integer;
alter table forward add column periodic_local_day_of_month integer;
alter table forward add column periodic_month integer;	
alter table forward add column periodic_local_month integer;	
alter table forward add column r_id integer;	

create table periodic (
	last_checked_time time
);
ALTER TABLE periodic OWNER TO ws;

alter table forward drop constraint forward_s_id_fkey;

create index log_org_idx on log (o_id);

-- Version 23.09

alter table mailout add column anonymous boolean;

CREATE SEQUENCE subevent_queue_seq START 1;
ALTER SEQUENCE subevent_queue_seq OWNER TO ws;

CREATE TABLE IF NOT EXISTS subevent_queue (
	id integer DEFAULT NEXTVAL('subevent_queue_seq') CONSTRAINT pk_subevent_queue PRIMARY KEY,
	ue_id integer,
	linkage_items text,    -- JSON
	status text,    -- new or failed
	reason text,	-- failure reason
	processed_time TIMESTAMP WITH TIME ZONE		-- Time of processing
	);
ALTER TABLE subevent_queue OWNER TO ws;

alter table upload_event add column processed_time timestamp with time zone;
create index idx_ue_processed_time on upload_event (processed_time);

alter table s3upload add column created_time timestamp with time zone;

alter table cms_alert add column filter text;

alter table server add column max_rate integer default 0;

-- Version 23.11
alter table question add column required_expression text;
alter table question add column readonly_expression text;

-- Version 24.01
alter table users add column basic_password text;
CREATE EXTENSION IF NOT EXISTS pgcrypto;

-- Version 24.02
create sequence email_id START 1;
ALTER SEQUENCE email_id OWNER TO ws;

alter table forward add column updated boolean;

alter table users add column current_survey_ident text;
update users set current_survey_ident = (select ident from survey where s_id = current_survey_id) where current_survey_ident is null and current_survey_id > 0;

-- Version 24.05
alter table users add column api_key text;

-- Version 24.06
CREATE UNLOGGED TABLE IF NOT EXISTS submission_queue
(
    element_identifier UUID PRIMARY KEY,
    time_inserted TIMESTAMP,
    ue_id integer,
    instanceid text,	-- Don't allow duplicates in the submission queue where they can be worked on in parallel
    restore boolean,
    payload JSON
);
ALTER TABLE submission_queue OWNER TO ws;

alter table upload_event add column queue_name text;
alter table upload_event add column queued boolean default false;
alter table upload_event add column restore boolean default false;

alter table subevent_queue add column created_time TIMESTAMP WITH TIME ZONE;

CREATE UNLOGGED TABLE IF NOT EXISTS monitor_data
(
    recorded_at TIMESTAMP WITH TIME ZONE,
    payload JSON
);
ALTER TABLE monitor_data OWNER TO ws;

CREATE UNLOGGED TABLE IF NOT EXISTS message_queue
(
    element_identifier UUID PRIMARY KEY,
    time_inserted TIMESTAMP,
    m_id integer,
    o_id integer,
    topic text,	
    description text,
    data text
);
ALTER TABLE message_queue OWNER TO ws;

alter table message add column queue_name text;
alter table message add column queued boolean default false;

-- Version 24.09
CREATE UNLOGGED TABLE IF NOT EXISTS key_queue
(
    element_identifier UUID PRIMARY KEY,
    key text,
    group_survey_ident text
);
ALTER TABLE key_queue OWNER TO ws;

alter table upload_event add column submission_type text;
alter table upload_event add column payload text;

CREATE TABLE IF NOT EXISTS sms_number (
    element_identifier UUID PRIMARY KEY,
    o_id integer,					-- Organisation that the number is allocated to
    time_modified TIMESTAMP WITH TIME ZONE,
    our_number text,			-- Our number that sends or receives messages
    survey_ident text,
    their_number_question text, -- The question in the survey that holds the number of the counterpart
    message_question text,		-- The question name in the survey that holds the message details
    description text
);
ALTER TABLE sms_number OWNER TO ws;
CREATE UNIQUE INDEX IF NOT EXISTS sms_number_to_idx ON sms_number(our_number);

ALTER TABLE server add column vonage_application_id text;
ALTER TABLE server add column vonage_webhook_secret text;

ALTER TABLE log  DROP CONSTRAINT log_o_id_fkey;

ALTER TABLE users add column app_key text;

ALTER TABLE organisation add column email_type text;
ALTER TABLE server add column email_type text;
update server set email_type = 'smtp' where email_type is null;

alter table organisation add column ft_force_token boolean default false;

alter table sms_number add column channel text;
update sms_number set channel = 'sms' where channel is null;
alter table record_event add column message text;

-- Version 24.10
--alter table survey_role add column group_survey_ident text;  -- Removed in 25.01

-- Version 24.11
ALTER TABLE organisation add column email_type text;
ALTER TABLE organisation add column aws_region text;
ALTER TABLE server add column aws_region text;
alter table sms_number add column mc_msg text;	-- Message to send if there is more than one case to update

-- Remove links security group
delete from groups where name = 'links';
delete from user_group where g_id = 13;

create index if not exists question_l_id_idx on question(l_id);	-- Address performance issue
create index if not exists form_table_name on form(table_name);
create index if not exists tasks_survey_idx on tasks(survey_ident);
create index if not exists linked_files_logical_path on linked_files(logical_path);

-- Console Admin
insert into groups(id,name) values(14,'console admin');

-- Improve Timezone Performance
CREATE TABLE IF NOT EXISTS timezone (
    name text,
    utc_offset text
);
ALTER TABLE timezone OWNER TO ws;

ALTER TABLE users add column total_tasks integer default -1;

alter table group_survey drop constraint group_survey_u_ident_fkey;			-- Remove constraint on group survey as sometimes this has to be null, for example when sub form is set up not group survey
delete from group_survey where group_ident is null and f_name is null;  	-- Remove unused values

-- Notification bundles
alter table forward add column bundle boolean default false;
alter table forward add column bundle_ident text;

-- Performance
create index if not exists survey_p_id on survey(p_id);

CREATE SEQUENCE bundle_seq START 1;
ALTER SEQUENCE bundle_seq OWNER TO ws;

CREATE TABLE if not exists bundle (
	id integer DEFAULT NEXTVAL('bundle_seq') CONSTRAINT pk_bundle PRIMARY KEY,
	group_survey_ident text,
	bundle_roles boolean,
	changed_by text,
	changed_ts TIMESTAMP WITH TIME ZONE	
	);
CREATE UNIQUE INDEX if not exists bundle_group_idx ON bundle(group_survey_ident);
ALTER TABLE bundle OWNER TO ws;

-- performance
create index message_created_idx on message (created_time);

-- remove dashboard security group
delete from groups where name = 'dashboard';
delete from user_group where g_id = 12;

-- Drop survey role constraint that causes issues when replacing a survey that has roles
alter table survey_role drop constraint survey_role_survey_ident_fkey;

-- Allow server setting to require security manager privilege to delete
alter table server add column sec_mgr_del boolean default false;

-- Add a flag to force reset of total tasks
alter table users add column reset_total_tasks boolean default false;

-- Add survey role groups
alter table survey_role add column role_group text default 'A';
update survey_role set role_group = 'A' where role_group is null;

-- Ensure users have a name
update users set name = ident where name = '' and not temporary;

-- Version 25.08
alter table subevent_queue add column thread text;

CREATE SEQUENCE notified_record_seq START 1;
ALTER SEQUENCE notified_record_seq OWNER TO ws;

CREATE TABLE public.notified_record (
	id integer default nextval('notified_record_seq') not null PRIMARY KEY,
	n_id integer REFERENCES forward(id) ON DELETE CASCADE,
	thread text
	);
ALTER TABLE notified_record OWNER TO ws;
CREATE INDEX n_thread ON notified_record(thread);

-- Remove foreign key on log archive file
alter table log_archive drop constraint log_archive_o_id_fkey;

-- Version 25.12
-- insert into groups(id,name) values(15,'mcp access');

-- Performance tuning
create index if not exists linked_forms_idx on linked_forms(linked_s_id);
delete from pending_message where created_time < now() - interval '1 year';
create index if not exists opted_in_people_idx on people(opted_in);
create index if not exists unsub_people_idx on people(unsubscribed);

-- Version 25.01 Multi-server subscriber support
-- Drop old indexes that interfere with query planner
drop index if exists idx_ue_ra;
drop index if exists idx_ue_queued;
drop index if exists idx_ue_applied;
-- Composite index for pending upload queries
create index concurrently if not exists idx_ue_pending on upload_event (results_db_applied, queued, incomplete, ue_id);

-- Add API rate limiting configuration
alter table server add column if not exists api_max_records integer default 0;

-- Performance improvement to get tasks
CREATE INDEX idx_tasks_tg_schedule_desc ON tasks(tg_id, schedule_at DESC);
delete from groups where id = 15;

-- Cloudflare Turnstile anti-bot support
alter table server add column if not exists turnstile_site_key text;
alter table server add column if not exists turnstile_secret_key text;
alter table survey add column if not exists turnstile boolean default false;

-- Self assign tasks
alter table task_rejected add column t_id integer REFERENCES tasks(id) ON DELETE CASCADE;
delete from task_rejected where t_id is null;

alter table dashboard_settings add column ds_chart_type text default 'histogram';
alter table dashboard_settings add column ds_show_meta boolean default true;
alter table dashboard_settings add column ds_wrap_text boolean default true;

-- Version 26.04 Subscriber worker identification
CREATE UNLOGGED TABLE IF NOT EXISTS subscriber_worker (
	id serial PRIMARY KEY,
	hostname text,
	pid bigint,
	subscriber_type text,
	queue_name text,
	started_time timestamptz DEFAULT now(),
	heartbeat timestamptz DEFAULT now()
);
ALTER TABLE subscriber_worker OWNER TO ws;

alter table upload_event add column if not exists worker_host text;
alter table s3upload add column if not exists worker_id text;
alter table message add column if not exists worker_host text;

-- Version 26.05 SharePoint integration
alter table server add column if not exists sharepoint_url text;
alter table server add column if not exists sharepoint_client_id text;
alter table server add column if not exists sharepoint_realm text;
alter table server add column if not exists sharepoint_cert_pem text;
alter table server add column if not exists sharepoint_auth_type text default 's2s';
alter table server add column if not exists sharepoint_username text;
alter table server add column if not exists sharepoint_password text;
alter table server add column if not exists sharepoint_domain text;

CREATE SEQUENCE IF NOT EXISTS sharepoint_list_map_seq START 1;
ALTER SEQUENCE sharepoint_list_map_seq OWNER TO ws;

CREATE TABLE IF NOT EXISTS sharepoint_list_map (
	id integer DEFAULT nextval('sharepoint_list_map_seq') NOT NULL PRIMARY KEY,
	o_id integer REFERENCES organisation(id) ON DELETE CASCADE,
	smap_name text NOT NULL,
	list_title text NOT NULL,
	refresh_minutes integer DEFAULT 60,
	last_sync TIMESTAMP WITH TIME ZONE,
	csv_table_id integer REFERENCES csvtable(id) ON DELETE SET NULL,
	enabled boolean DEFAULT true
	);
CREATE INDEX IF NOT EXISTS sharepoint_list_map_org_idx ON sharepoint_list_map(o_id);
ALTER TABLE sharepoint_list_map OWNER TO ws;

-- Version 26.04 Workflow node positions per user per organisation
CREATE TABLE IF NOT EXISTS workflow_node_positions (
	user_ident   text,
	o_id         integer references organisation(id) on delete cascade,
	positions    jsonb,
	PRIMARY KEY (user_ident, o_id)
);
ALTER TABLE workflow_node_positions OWNER TO ws;
-- Version 26.04.1 Explicit workflow starting-point forms
CREATE TABLE IF NOT EXISTS workflow_start (
    id      serial PRIMARY KEY,
    s_ident text,
    p_id    integer references project(id) on delete cascade
);
ALTER TABLE workflow_start OWNER TO ws;
-- Workflow explicit sequence: predecessor node ID set by workflow canvas
ALTER TABLE forward     ADD COLUMN IF NOT EXISTS wf_prev_node_id text;
ALTER TABLE task_group  ADD COLUMN IF NOT EXISTS wf_prev_node_id text;
-- Version 26.04.2 Prevent duplicate submissions in queue via unique index
create unique index if not exists submission_queue_instanceid_idx on submission_queue(instanceid);

-- Show form index panel in webforms
alter table survey add column if not exists show_form_index boolean default false;

-- Allow notifications to be sent from WebForms (per org)
alter table organisation add column if not exists notification_webform boolean default false;
-- Allow redactions (per org)
alter table organisation add column if not exists enable_redact boolean default false;

-- PII / anonymise flag on questions
alter table question add column if not exists pii text;

-- DPO role
insert into groups(id,name) values(16,'dpo') on conflict(id) do nothing;

-- Version 26.05 Email reply tracking
alter table server add column if not exists email_response_bucket text;
alter table server add column if not exists email_response_domain text;
alter table notification_log add column if not exists aws_message_id text;
alter table record_event add column if not exists ses_message_id text;
create unique index if not exists record_event_ses_message_id_idx on record_event(ses_message_id) where ses_message_id is not null;

-- Referenced cases: denormalised index of records a user owns (case) or can reference (read only)
CREATE SEQUENCE IF NOT EXISTS record_user_id_seq START 1;
ALTER SEQUENCE record_user_id_seq OWNER TO ws;
CREATE TABLE IF NOT EXISTS record_user (
	id bigint NOT NULL DEFAULT nextval('record_user_id_seq') PRIMARY KEY,
	assignee integer REFERENCES users(id) ON DELETE CASCADE,
	assignee_ident text NOT NULL,
	group_survey_ident text NOT NULL,
	survey_ident text,
	thread text NOT NULL,
	access text NOT NULL DEFAULT 'owner',	-- owner || reference
	read_only boolean NOT NULL DEFAULT false,
	created_by text,
	created_at timestamp with time zone DEFAULT now()
);
CREATE UNIQUE INDEX IF NOT EXISTS record_user_unique ON record_user(assignee_ident, group_survey_ident, thread);
CREATE INDEX IF NOT EXISTS record_user_assignee ON record_user(assignee_ident);
CREATE INDEX IF NOT EXISTS record_user_thread ON record_user(group_survey_ident, thread);
ALTER TABLE record_user OWNER TO ws;
-- One off backfill of record_user owner rows from _assigned, run by the forward subscriber on startup
alter table server add column if not exists record_user_backfilled boolean default false;

-- Operations Monitor per-organisation settings (stale interval, RAG thresholds, trend window)
CREATE TABLE IF NOT EXISTS ops_settings (
	o_id        integer PRIMARY KEY references organisation(id) on delete cascade,
	settings    jsonb,
	changed_by  text,
	changed_ts  timestamp with time zone
);
ALTER TABLE ops_settings OWNER TO ws;

-- Remove the unused per-user alert table (never populated; ops alerts now come from
-- cms_alert + case_alert_triggered, alert history lives in record_event)
DROP TABLE IF EXISTS alert CASCADE;
DROP SEQUENCE IF EXISTS alert_seq CASCADE;

-- Cap the number of records supplied by a survey when used as a reference file.
-- 0 (default) = unlimited; N = only the latest N records by _upload_time.
alter table survey add column if not exists max_reference_records integer default 0;
-- Record the cap a linked file was generated with so it can be regenerated when the cap changes.
alter table linked_forms add column if not exists max_records integer default 0;

-- Static pseudo-SQL filter restricting the reference data a survey bundle pulls from a source
-- survey.  Defined at the group level (like roles) per (linker group, source group) pair.
create sequence if not exists reference_filter_seq start 1;
alter sequence reference_filter_seq owner to ws;
create table if not exists reference_filter (
	id integer default nextval('reference_filter_seq') constraint pk_reference_filter primary key,
	linker_s_ident text not null,		-- requesting survey group ident
	linked_s_ident text not null,		-- source survey group ident
	filter text,
	enabled boolean default true
);
alter table reference_filter owner to ws;
create unique index if not exists reference_filter_idx on reference_filter(linker_s_ident, linked_s_ident);
-- Cap the number of records supplied over this connection.  0 (default) = unlimited.
-- Replaces the shortcut survey.max_reference_records (now dormant) with a per-connection cap.
alter table reference_filter add column if not exists max_records integer default 0;
