diff --git a/app/CHANGELOG.md b/app/CHANGELOG.md
index 23607055..17a3a6ad 100644
--- a/app/CHANGELOG.md
+++ b/app/CHANGELOG.md
@@ -7,6 +7,7 @@
- Remove faulty KPIs from results view [[#457](https://github.com/open-plan-tool/gui/pull/457)]
- Fix certain fields in results display appearing blank for Chromium browsers [[#461](https://github.com/open-plan-tool/gui/pull/461)]
- Update scenario simulation status on project overview [[#459](https://github.com/open-plan-tool/gui/pull/459)]
+- Improve timeseries file upload display and error handling [[#452](https://github.com/open-plan-tool/gui/pull/452)]
## [v2.1.1] – 2026-04-15
### Fixed
diff --git a/app/projects/forms.py b/app/projects/forms.py
index a3848349..5bda2acb 100644
--- a/app/projects/forms.py
+++ b/app/projects/forms.py
@@ -966,9 +966,15 @@ def clean(self):
input_method = ts_data["input_method"]["type"]
if input_method == TS_UPLOAD_TYPE or input_method == TS_MANUAL_TYPE:
# replace the dict with a new timeseries instance
- cleaned_data["input_timeseries"] = self.assign_timeseries_from_input(
- ts_data
- )
+ timeseries_obj = self.assign_timeseries_from_input(ts_data)
+ num_timestamps = self.scenario.get_num_timesteps
+ if len(timeseries_obj.values) != num_timestamps:
+ msg = _(
+ f"The length of the timeseries ({len(timeseries_obj.values)}) does not match the lentgh of the selected timesteps ({num_timestamps}). You can check your timeseries file or change the timesteps in step 1."
+ )
+ self.add_error("input_timeseries", msg)
+ else:
+ cleaned_data["input_timeseries"] = timeseries_obj
if input_method == TS_SELECT_TYPE:
# return the timeseries instance
timeseries_id = ts_data["input_method"]["extra_info"]
diff --git a/app/projects/helpers.py b/app/projects/helpers.py
index 9be1e324..e2fe3574 100644
--- a/app/projects/helpers.py
+++ b/app/projects/helpers.py
@@ -438,7 +438,15 @@ def clean(self, values):
timeseries_id = ""
if timeseries_file is not None:
- input_timeseries_values = parse_input_timeseries(timeseries_file)
+ try:
+ input_timeseries_values = parse_input_timeseries(timeseries_file)
+ except (ValueError, TypeError) as e:
+ self.set_widget_error()
+ raise ValidationError(str(e))
+ except Exception as e:
+ self.set_widget_error()
+ raise ValidationError(f"Could not parse uploaded file: {e}")
+
answer = input_timeseries_values
input_dict = dict(type=TS_UPLOAD_TYPE, extra_info=timeseries_file.name)
elif timeseries_id != "":
@@ -534,27 +542,76 @@ def set_widget_error(self):
def parse_csv_timeseries(file_str):
io_string = io.StringIO(file_str)
delimiter = ","
+ is_comma_decimal = False
+ msg = "The uploaded file has an invalid format. Please provide a CSV with 1 or 2 columns containing only numeric values (no header). If two columns are used, the first must be the index (e.g., timestamps) and the second the corresponding values."
+
+ lines = file_str.splitlines()
+ # check if there are timestamps
+ has_timestamp = any(":" in line or "-" in line for line in lines)
+
+ # --- delimiter detection ---
if file_str.count(";") > 0:
delimiter = ";"
- # check if the number of , is an integer time the number of line return
- # if not, the , is probably not a column separator and a decimal separator indeed
- if file_str.count(",") % (file_str.count("\n") + 1) != 0:
- delimiter = ";"
+ # --- comma ambiguity ---
+ # If commas exist, we need to distinguish:
+ # (A) CSV with comma delimiter
+ # (B) decimal comma (single column numeric data)
+
+ if delimiter == ",":
+ comma_per_line = [line.count(",") for line in lines if line.strip()]
+ if comma_per_line and all(c == 1 for c in comma_per_line):
+ if file_str.count(".") > 0:
+ # if all lines contain a "," AND there is a "." in the file, strong indicator for "," delimiter
+ delimiter = ","
+ elif not has_timestamp:
+ raise ValidationError(msg)
+ else:
+ # safe to assume decimal comma in single-column case
+ if comma_per_line and all(c <= 1 for c in comma_per_line):
+ is_comma_decimal = True
+ delimiter = ";"
+
+ # check for number of columns, throw error if more then 2
+ if any(len(line.split(delimiter)) > 2 for line in lines if line.strip()):
+ raise ValidationError(msg)
+
+ # --- parsing ---
reader = csv.reader(io_string, delimiter=delimiter)
timeseries_values = []
+
for row in reader:
+ if not row:
+ continue
+
if len(row) == 1:
value = row[0]
else:
- # assumes the first row is timestamps and read the second one, ignore any other row
- value = row[1]
- # convert potential comma used as decimal point to decimal point
- timeseries_values.append(float(value.replace(",", ".")))
+ # since we have more than 1 col, check if timeseries is only in the first, if not raise error
+ if is_timestamp(row[-1]):
+ raise ValidationError(msg)
+ value = row[-1]
+ value = value.strip()
+
+ # --- decimal normalization ---
+ if is_comma_decimal:
+ value = value.replace(",", ".")
+ else:
+ if "," in value and "." not in value:
+ value = value.replace(",", ".")
+ if value.isalpha():
+ # catch if there is a header, then the file cannot be parsed
+ raise ValidationError(msg)
+ timeseries_values.append(float(value))
return timeseries_values
+def is_timestamp(values):
+ # checks if there is a timestamp in the given value/values
+ return ":" in values or "-" in values
+
+
def parse_xlsx_timeseries(file_buffer):
wb = load_workbook(filename=file_buffer)
worksheet = wb.active
diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py
index 979d4806..04fdfd6b 100644
--- a/app/projects/models/base_models.py
+++ b/app/projects/models/base_models.py
@@ -788,7 +788,7 @@ def timestamps(self):
@property
def input_timeseries_values(self):
if self.is_input_timeseries_empty() is False:
- answer = json.loads(self.input_timeseries)
+ answer = self.input_timeseries.get_values
else:
answer = []
return answer
@@ -961,7 +961,7 @@ def export(self, connections=False):
return dm
def is_input_timeseries_empty(self):
- return self.input_timeseries == ""
+ return self.input_timeseries is None
class COPCalculator(models.Model):
diff --git a/app/projects/tests.py b/app/projects/tests.py
index 1c4ef73e..fc7693a4 100644
--- a/app/projects/tests.py
+++ b/app/projects/tests.py
@@ -1,3 +1,4 @@
+import datetime
import json
import pytest
@@ -245,78 +246,196 @@ def test_export_project_via_get_with_scenarios(self):
self.assertIn("scenario_set_data", response.json())
-# class UploadTimeseriesTest(TestCase):
-# fixtures = ["fixtures/benchmarks_fixture.json"]
-#
-# @classmethod
-# def setUpTestData(cls):
-# pass
-#
-# def setUp(self):
-# self.factory = RequestFactory()
-# self.client.login(username="testUser", password="ASas12,.")
-# self.project = Project.objects.get(id=1)
-# self.post_url = reverse("asset_create_or_update", args=[2, "demand"])
-#
-# def test_load_demand_csv_double_timeseries(self):
-# with open("./test_files/test_ts_double.csv") as fp:
-# data = {
-# "name": "Test_input_timeseries",
-# "pos_x": 0,
-# "pos_y": 0,
-# "input_timeseries": fp,
-# }
-# response = self.client.post(self.post_url, data, format="multipart")
-# self.assertEqual(response.status_code, 200)
-# asset = Asset.objects.last()
-# self.assertEqual(asset.input_timeseries_values, [1, 2, 3, 4])
-#
-# def test_load_demand_csv_double_decimal_point_with_comma(self):
-# with open("./test_files/test_ts_csv_semicolon.csv") as fp:
-# data = {
-# "name": "Test_input_timeseries",
-# "pos_x": 0,
-# "pos_y": 0,
-# "input_timeseries": fp,
-# }
-# response = self.client.post(self.post_url, data, format="multipart")
-# self.assertEqual(response.status_code, 200)
-# asset = Asset.objects.last()
-# self.assertEqual(asset.input_timeseries_values, [8.5, 3.3, 4.0, 6.0])
-#
-# def test_load_demand_xlsx_double_timeseries(self):
-# with open("./test_files/test_ts_double.xlsx", "rb") as fp:
-# data = {
-# "name": "Test_input_timeseries",
-# "pos_x": 0,
-# "pos_y": 0,
-# "input_timeseries": fp,
-# }
-# response = self.client.post(self.post_url, data, format="multipart")
-# self.assertEqual(response.status_code, 200)
-# asset = Asset.objects.last()
-# self.assertEqual(asset.input_timeseries_values, [1, 2, 3, 4])
-#
-# def test_load_demand_csv_decimal_point_with_comma(self):
-# with open("./test_files/test_ts_comma_decimal.csv") as fp:
-# data = {
-# "name": "Test_input_timeseries",
-# "pos_x": 0,
-# "pos_y": 0,
-# "input_timeseries": fp,
-# }
-# response = self.client.post(self.post_url, data, format="multipart")
-# self.assertEqual(response.status_code, 200)
-# asset = Asset.objects.last()
-# self.assertEqual(asset.input_timeseries_values, [1.2, 2, 3.0, 4])
-#
-# def test_load_demand_file_wrong_format_raises_error(self):
-# with open("./test_files/test_ts.notsupported") as fp:
-# data = {
-# "name": "Test_input_timeseries",
-# "pos_x": 0,
-# "pos_y": 0,
-# "input_timeseries": fp,
-# }
-# response = self.client.post(self.post_url, data, format="multipart")
-# self.assertEqual(response.status_code, 422)
+class UploadTimeseriesTest(TestCase):
+ fixtures = ["fixtures/benchmarks_fixture.json"]
+
+ @classmethod
+ def setUpTestData(cls):
+ pass
+
+ def setUp(self):
+ self.factory = RequestFactory()
+ self.client.login(username="testUser", password="ASas12,.")
+ self.project = Project.objects.get(id=1)
+
+ # set up scenario for timeseries lengths of 4
+ self.scenario = self.project.scenario_set.first()
+ self.scenario.time_step = 360 # 6 hours
+ self.scenario.evaluated_period = 1
+ self.scenario.start_date = datetime.datetime(2020, 1, 1)
+ self.scenario.save()
+
+ self.post_url = reverse("asset_create_or_update", args=[2, "demand"])
+
+ def test_load_demand_csv_timestamp_format(self):
+ with open("./test_files/test_ts_timestamp_format.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [1, 2, 3, 4])
+
+ def test_load_demand_csv_timestamp_format_reverse_raises_error(self):
+ with open("./test_files/test_ts_timestamp_format_reverse.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ form = response.context["form"]
+ self.assertIn("input_timeseries", form.errors)
+ self.assertIn("invalid format", str(form.errors["input_timeseries"]))
+ self.assertEqual(response.status_code, 422)
+
+ def test_load_demand_csv_semicolon_format_decimal_comma(self):
+ with open("./test_files/test_ts_semicolon_format_decimal_comma.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [8.5, 3.3, 4.0, 6.0])
+
+ def test_load_demand_csv_semicolon_format_decimal_point(self):
+ with open("./test_files/test_ts_semicolon_format_decimal_point.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [8.5, 3.3, 4.0, 6.0])
+
+ def test_load_demand_csv_comma_format_decimal_point(self):
+ with open("./test_files/test_ts_comma_format_decimal_point.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [8.5, 3.3, 4.0, 6.0])
+
+ def test_load_demand_xlsx_double_timeseries(self):
+ with open("./test_files/test_ts_double.xlsx", "rb") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [1, 2, 3, 4])
+
+ def test_load_demand_csv_1col_format_decimal_comma(self):
+ with open("./test_files/test_ts_1col_format_decimal_comma.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [1.2, 2, 3.0, 4])
+
+ def test_load_demand_csv_1col_format_decimal_point(self):
+ with open("./test_files/test_ts_1col_format_decimal_point.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ self.assertEqual(response.status_code, 200)
+ asset = Asset.objects.last()
+ self.assertEqual(asset.input_timeseries_values, [1.2, 2, 3.0, 4])
+
+ def test_load_demand_file_wrong_format_raises_error(self):
+ with open("./test_files/test_ts_wrong_format.notsupported") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ form = response.context["form"]
+ self.assertIn("input_timeseries", form.errors)
+ self.assertIn("not supported", str(form.errors["input_timeseries"]))
+ self.assertEqual(response.status_code, 422)
+
+ def test_load_demand_csv_semicolon_header_format_raises_error(self):
+ with open(
+ "./test_files/test_ts_semicolon_header_format_decimal_point.csv"
+ ) as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ form = response.context["form"]
+ self.assertIn("input_timeseries", form.errors)
+ self.assertIn("invalid format", str(form.errors["input_timeseries"]))
+ self.assertEqual(response.status_code, 422)
+
+ def test_load_demand_csv_timeseries_timestep_length_mismatch_raises_error(self):
+ with open("./test_files/test_ts_length_mismatch.csv") as fp:
+ data = {
+ "name": "Test_input_timeseries",
+ "pos_x": 0,
+ "pos_y": 0,
+ "input_timeseries_scalar": "",
+ "input_timeseries_select": "",
+ "input_timeseries_file": fp,
+ }
+ response = self.client.post(self.post_url, data, format="multipart")
+ form = response.context["form"]
+ self.assertIn("input_timeseries", form.errors)
+ self.assertIn(
+ "does not match the lentgh", str(form.errors["input_timeseries"])
+ )
+ self.assertEqual(response.status_code, 422)
diff --git a/app/static/js/traceplot.js b/app/static/js/traceplot.js
index 1a579270..cc0ae18b 100644
--- a/app/static/js/traceplot.js
+++ b/app/static/js/traceplot.js
@@ -45,9 +45,6 @@ function makePlotly( x, y, plot_id="",userLayout=null){
if(ts_timestamps.length == y.length){
x = ts_timestamps
}
- else{
- alert("The number of values in your uploaded timeseries (" + y.length + ") does not match the scenario timestamps (" + ts_timestamps.length + ").\nPlease change the scenario settings or upload a new timeseries")
- }
}
var plotLayout = {...layout};
@@ -318,7 +315,6 @@ function parseExcelData(data){
});
}
else{
-
reader.onload = loadHandler;
// Read file into memory as UTF-8
reader.readAsText(fileToRead);
@@ -326,9 +322,16 @@ function parseExcelData(data){
function loadHandler(event) {
- var csv = event.target.result;
- d3array = d3.csvParseRows(csv);
- processData(d3array);
+ var csv = event.target.result;
+ const lines = csv.split('\n');
+ const comma_per_line = lines.map(line => (line.match(/,/g) || []).length);
+ if (comma_per_line.length > 0 && comma_per_line.every(c => c <= 1) && !csv.includes(".") || csv.includes(";")) {
+ // Safe to assume decimal comma in single-column case or semicolon as delimiter in csv
+ d3array = d3.dsvFormat(";").parseRows(csv);
+ } else {
+ d3array = d3.csvParseRows(csv);
+ }
+ processData(d3array);
}
diff --git a/app/templates/asset/asset_create_form.html b/app/templates/asset/asset_create_form.html
index 7ca93d82..484bd4ba 100644
--- a/app/templates/asset/asset_create_form.html
+++ b/app/templates/asset/asset_create_form.html
@@ -80,6 +80,12 @@ {% translate "Technical parameters" %}
{{ form|get_field:"input_timeseries"|as_crispy_field }}
+ {% for error in form.input_timeseries.errors %}
+
+ {% endfor %}
+
{% if input_timeseries_timestamps %}
diff --git a/app/test_files/test_ts_comma_decimal.csv b/app/test_files/test_ts_1col_format_decimal_comma.csv
similarity index 100%
rename from app/test_files/test_ts_comma_decimal.csv
rename to app/test_files/test_ts_1col_format_decimal_comma.csv
diff --git a/app/test_files/test_ts_1col_format_decimal_point.csv b/app/test_files/test_ts_1col_format_decimal_point.csv
new file mode 100755
index 00000000..9d3a0be4
--- /dev/null
+++ b/app/test_files/test_ts_1col_format_decimal_point.csv
@@ -0,0 +1,4 @@
+1.2
+2
+3.0
+4
diff --git a/app/test_files/test_ts_comma_format_decimal_point.csv b/app/test_files/test_ts_comma_format_decimal_point.csv
new file mode 100755
index 00000000..968fa502
--- /dev/null
+++ b/app/test_files/test_ts_comma_format_decimal_point.csv
@@ -0,0 +1,4 @@
+1,8.5
+2,3.3
+3,4
+4,6.0
diff --git a/app/test_files/test_ts_length_mismatch.csv b/app/test_files/test_ts_length_mismatch.csv
new file mode 100755
index 00000000..697a9b1c
--- /dev/null
+++ b/app/test_files/test_ts_length_mismatch.csv
@@ -0,0 +1,5 @@
+1,8.5
+2,3.3
+3,4
+4,6.0
+5,4.0
diff --git a/app/test_files/test_ts_csv_semicolon.csv b/app/test_files/test_ts_semicolon_format_decimal_comma.csv
similarity index 100%
rename from app/test_files/test_ts_csv_semicolon.csv
rename to app/test_files/test_ts_semicolon_format_decimal_comma.csv
diff --git a/app/test_files/test_ts_semicolon_format_decimal_point.csv b/app/test_files/test_ts_semicolon_format_decimal_point.csv
new file mode 100755
index 00000000..cb5c93e1
--- /dev/null
+++ b/app/test_files/test_ts_semicolon_format_decimal_point.csv
@@ -0,0 +1,4 @@
+1;8.5
+2;3.3
+3;4
+4;6.0
diff --git a/app/test_files/test_ts_semicolon_header_format_decimal_point.csv b/app/test_files/test_ts_semicolon_header_format_decimal_point.csv
new file mode 100755
index 00000000..843c0d9b
--- /dev/null
+++ b/app/test_files/test_ts_semicolon_header_format_decimal_point.csv
@@ -0,0 +1,5 @@
+timestep;value
+1;8.5
+2;3.3
+3;4
+4;6.0
diff --git a/app/test_files/test_ts_double.csv b/app/test_files/test_ts_timestamp_format.csv
similarity index 100%
rename from app/test_files/test_ts_double.csv
rename to app/test_files/test_ts_timestamp_format.csv
diff --git a/app/test_files/test_ts_timestamp_format_reverse.csv b/app/test_files/test_ts_timestamp_format_reverse.csv
new file mode 100755
index 00000000..a906c0e5
--- /dev/null
+++ b/app/test_files/test_ts_timestamp_format_reverse.csv
@@ -0,0 +1,4 @@
+1,"2020-01-01 10:00:00"
+2,"2020-01-01 11:00:00"
+3,"2020-01-01 12:00:00"
+4,"2020-01-01 13:00:00"
diff --git a/app/test_files/test_ts.notsupported b/app/test_files/test_ts_wrong_format.notsupported
similarity index 100%
rename from app/test_files/test_ts.notsupported
rename to app/test_files/test_ts_wrong_format.notsupported