Coverage for enpt_enmapboxapp/_enpt_alg

1# -*- coding: utf-8 -*-

3# enpt_enmapboxapp, A QGIS EnMAPBox plugin providing a GUI for the EnMAP processing tools (EnPT)

7# This software was developed within the context of the EnMAP project supported

8# by the DLR Space Administration with funds of the German Federal Ministry of

9# Economic Affairs and Energy (on the basis of a decision by the German Bundestag:

10# 50 EE 1529) and contributions from DLR, GFZ and OHB System AG.

11#

12# This program is free software: you can redistribute it and/or modify it under

13# the terms of the GNU Lesser General Public License as published by the Free

14# Software Foundation, either version 3 of the License, or (at your option) any

15# later version.

16#

17# This program is distributed in the hope that it will be useful, but WITHOUT

18# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

19# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more

20# details.

21#

22# You should have received a copy of the GNU Lesser General Public License along

23# with this program. If not, see <https://www.gnu.org/licenses/>.

25"""This module provides the base class for EnPTAlgorithm and ExternalEnPTAlgorithm."""

27import os

28from os.path import expanduser

29import psutil

30from importlib.util import find_spec

31from datetime import date

32from multiprocessing import cpu_count

33from threading import Thread

34from queue import Queue

35from subprocess import Popen, PIPE

36from glob import glob

37from warnings import warn

39from qgis.core import (

40 QgsProcessingAlgorithm,

41 QgsProcessingParameterFile,

42 QgsProcessingParameterNumber,

43 QgsProcessingParameterFolderDestination,

44 QgsProcessingParameterBoolean,

45 QgsProcessingParameterDefinition,

46 QgsProcessingParameterRasterLayer,

47 QgsProcessingParameterEnum,

48 NULL

49)

51from .version import __version__

54class _EnPTBaseAlgorithm(QgsProcessingAlgorithm):

55 # NOTE: The parameter assignments made here follow the parameter names in enpt/options/options_schema.py

57 # Input parameters

58 P_json_config = 'json_config'

59 P_CPUs = 'CPUs'

60 P_path_l1b_enmap_image = 'path_l1b_enmap_image'

61 P_path_l1b_enmap_image_gapfill = 'path_l1b_enmap_image_gapfill'

62 P_path_dem = 'path_dem'

63 P_average_elevation = 'average_elevation'

64 P_output_dir = 'output_dir'

65 P_working_dir = 'working_dir'

66 P_n_lines_to_append = 'n_lines_to_append'

67 P_drop_bad_bands = 'drop_bad_bands'

68 P_disable_progress_bars = 'disable_progress_bars'

69 P_output_format = 'output_format'

70 P_output_interleave = 'output_interleave'

71 P_output_nodata_value = 'output_nodata_value'

72 P_path_earthSunDist = 'path_earthSunDist'

73 P_path_solar_irr = 'path_solar_irr'

74 P_scale_factor_toa_ref = 'scale_factor_toa_ref'

75 P_enable_keystone_correction = 'enable_keystone_correction'

76 P_enable_vnir_swir_coreg = 'enable_vnir_swir_coreg'

77 P_path_reference_image = 'path_reference_image'

78 P_enable_ac = 'enable_ac'

79 P_mode_ac = 'mode_ac'

80 P_polymer_additional_results = 'polymer_additional_results'

81 P_polymer_root = 'polymer_root'

82 P_threads = 'threads'

83 P_blocksize = 'blocksize'

84 P_auto_download_ecmwf = 'auto_download_ecmwf'

85 P_scale_factor_boa_ref = 'scale_factor_boa_ref'

86 P_run_smile_P = 'run_smile_P'

87 P_run_deadpix_P = 'run_deadpix_P'

88 P_deadpix_P_algorithm = 'deadpix_P_algorithm'

89 P_deadpix_P_interp_spectral = 'deadpix_P_interp_spectral'

90 P_deadpix_P_interp_spatial = 'deadpix_P_interp_spatial'

91 P_ortho_resampAlg = 'ortho_resampAlg'

92 P_vswir_overlap_algorithm = 'vswir_overlap_algorithm'

93 P_target_projection_type = 'target_projection_type'

94 P_target_epsg = 'target_epsg'

96 # # Output parameters

97 P_OUTPUT_RASTER = 'outraster'

98 # P_OUTPUT_VECTOR = 'outvector'

99 # P_OUTPUT_FILE = 'outfile'

100 P_OUTPUT_FOLDER = 'outfolder'

101

102 def group(self):

103 return 'Pre-Processing'

104

105 def groupId(self):

106 return 'PreProcessing'

107

108 def name(self):

109 return 'EnPTAlgorithm'

110

111 def displayName(self):

112 return f'EnPT - EnMAP Processing Tool (v{__version__})'

113

114 def createInstance(self, *args, **kwargs):

115 return type(self)()

116

117 @staticmethod

118 def _get_default_polymer_root():

119 if not find_spec('polymer'):

120 return ''

121 elif not find_spec('polymer').origin:

122 # this, e.g., happens when installing POLYMER with 'pip install' instead of 'pip install -e'

123 print('POLYMER package found, but it is not correctly installed.')

124 warn('POLYMER does not seem to be correctly installed. Find the installation instructions here: '

125 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/'

126 'installation.html#optional-install-polymer-for-advanced-atmospheric-correction-over-water-surfaces')

127 return ''

128 else:

129 return os.path.abspath(os.path.join(os.path.dirname(find_spec('polymer').origin), os.pardir))

130

131 @staticmethod

132 def _get_default_output_dir():

133 userhomedir = expanduser('~')

134

135 default_enpt_dir = \

136 os.path.join(userhomedir, 'Documents', 'EnPT', 'Output') if os.name == 'nt' else\

137 os.path.join(userhomedir, 'EnPT', 'Output')

138

139 outdir_nocounter = os.path.join(default_enpt_dir, date.today().strftime('%Y%m%d'))

140

141 counter = 1

142 while os.path.isdir('%s__%s' % (outdir_nocounter, counter)):

143 counter += 1

144

145 return '%s__%s' % (outdir_nocounter, counter)

146

147 def addParameter(self, param, *args, advanced=False, **kwargs):

148 """Add a parameter to the QgsProcessingAlgorithm.

149

150 This overrides the parent method to make it accept an 'advanced' parameter.

151

152 :param param: the parameter to be added

153 :param args: arguments to be passed to the parent method

154 :param advanced: whether the parameter should be flagged as 'advanced'

155 :param kwargs: keyword arguments to be passed to the parent method

156 """

157 if advanced:

158 param.setFlags(param.flags() | QgsProcessingParameterDefinition.FlagAdvanced)

159

160 super(_EnPTBaseAlgorithm, self).addParameter(param, *args, **kwargs)

161

162 def initAlgorithm(self, configuration=None):

163 self.addParameter(

164 QgsProcessingParameterFile(

165 name=self.P_json_config,

166 description='Configuration JSON template file',

167 behavior=QgsProcessingParameterFile.File,

168 extension='json',

169 defaultValue=None,

170 optional=True))

171

172 self.addParameter(

173 QgsProcessingParameterNumber(

174 name=self.P_CPUs,

175 description='Number of CPU cores to be used for processing',

176 type=QgsProcessingParameterNumber.Integer,

177 defaultValue=cpu_count(), minValue=0, maxValue=cpu_count()),

178 advanced=True)

179

180 self.addParameter(

181 QgsProcessingParameterFile(

182 name=self.P_path_l1b_enmap_image,

183 description='EnMAP Level-1B image (zip-archive or root directory)'))

184

185 self.addParameter(

186 QgsProcessingParameterFile(

187 name=self.P_path_l1b_enmap_image_gapfill,

188 description='Adjacent EnMAP Level-1B image to be used for gap-filling (zip-archive or root directory)',

189 optional=True),

190 advanced=True)

191

192 self.addParameter(

193 QgsProcessingParameterRasterLayer(

194 name=self.P_path_dem,

195 description='Input path of digital elevation model in map or sensor geometry; GDAL compatible file '

196 'format \n(must cover the EnMAP L1B data completely if given in map geometry or must have '

197 'the same \npixel dimensions like the EnMAP L1B data if given in sensor geometry)',

198 optional=True))

199

200 self.addParameter(

201 QgsProcessingParameterNumber(

202 name=self.P_average_elevation,

203 description='Average elevation in meters above sea level \n'

204 '(may be provided if no DEM is available and ignored if DEM is given)',

205 type=QgsProcessingParameterNumber.Integer,

206 defaultValue=0),

207 advanced=True)

208

209 self.addParameter(

210 QgsProcessingParameterFolderDestination(

211 name=self.P_output_dir,

212 description='Output directory where processed data and log files are saved',

213 defaultValue=self._get_default_output_dir(),

214 optional=True))

215

216 self.addParameter(

217 QgsProcessingParameterFile(

218 name=self.P_working_dir,

219 description='Directory to be used for temporary files',

220 behavior=QgsProcessingParameterFile.Folder,

221 defaultValue=None,

222 optional=True))

223

224 self.addParameter(

225 QgsProcessingParameterNumber(

226 name=self.P_n_lines_to_append,

227 description='Number of lines to be added to the main image [if not given, use the whole imgap]',

228 type=QgsProcessingParameterNumber.Integer,

229 defaultValue=None,

230 optional=True),

231 advanced=True)

232

233 self.addParameter(

234 QgsProcessingParameterBoolean(

235 name=self.P_drop_bad_bands,

236 description='Do not include bad bands (water absorption bands 1358-1453 nm / 1814-1961 nm) '

237 'in the L2A product',

238 defaultValue=True),

239 advanced=True)

240

241 self.addParameter(

242 QgsProcessingParameterBoolean(

243 name=self.P_disable_progress_bars,

244 description='Disable all progress bars during processing',

245 defaultValue=True),

246 advanced=True)

247

248 self.addParameter(

249 QgsProcessingParameterEnum(

250 name=self.P_output_format,

251 description="Output format (file format of all raster output files)",

252 options=['GTiff', 'ENVI'],

253 defaultValue=0),

254 advanced=True)

255

256 self.addParameter(

257 QgsProcessingParameterEnum(

258 name=self.P_output_interleave,

259 description="Output raster data interleaving type",

260 options=['band (BSQ)', 'line (BIL)', 'pixel (BIP)'],

261 defaultValue=2),

262 advanced=True)

263

264 self.addParameter(

265 QgsProcessingParameterNumber(

266 name=self.P_output_nodata_value,

267 description="Output no-data/background value (should be within the signed integer 16-bit range)",

268 type=QgsProcessingParameterNumber.Integer,

269 defaultValue=-32768,

270 optional=True),

271 advanced=True)

272

273 self.addParameter(

274 QgsProcessingParameterFile(

275 name=self.P_path_earthSunDist,

276 description='Input path of the earth sun distance model',

277 defaultValue=None,

278 optional=True),

279 advanced=True)

280

281 self.addParameter(

282 QgsProcessingParameterFile(

283 name=self.P_path_solar_irr,

284 description='Input path of the solar irradiance model',

285 defaultValue=None,

286 optional=True),

287 advanced=True)

288

289 self.addParameter(

290 QgsProcessingParameterNumber(

291 name=self.P_scale_factor_toa_ref,

292 description='Scale factor to be applied to TOA reflectance result',

293 type=QgsProcessingParameterNumber.Integer,

294 defaultValue=10000, minValue=0),

295 advanced=True)

296

297 # self.addParameter(

298 # QgsProcessingParameterBoolean(

299 # name=self.P_enable_keystone_correction,

300 # description='Keystone correction',

301 # defaultValue=False))

302

303 # self.addParameter(

304 # QgsProcessingParameterBoolean(

305 # name=self.P_enable_vnir_swir_coreg,

306 # description='VNIR/SWIR co-registration',

307 # defaultValue=False))

308

309 self.addParameter(

310 QgsProcessingParameterRasterLayer(

311 name=self.P_path_reference_image,

312 description='Reference image for absolute co-registration.',

313 defaultValue=None,

314 optional=True))

315

316 self.addParameter(

317 QgsProcessingParameterBoolean(

318 name=self.P_enable_ac,

319 description='Enable atmospheric correction',

320 defaultValue=True))

321

322 self.addParameter(

323 QgsProcessingParameterEnum(

324 name=self.P_mode_ac,

325 description="Atmospheric correction mode",

326 options=['land - SICOR is applied to land AND water',

327 'water - POLYMER is applied to water only; land is cleared ',

328 'combined - SICOR is applied to land and POLYMER to water'],

329 defaultValue=2))

330

331 self.addParameter(

332 QgsProcessingParameterBoolean(

333 name=self.P_polymer_additional_results,

334 description="Enable generation of additional results from ACwater/POLYMER (if executed)",

335 defaultValue=True))

336

337 self.addParameter(

338 QgsProcessingParameterFile(

339 name=self.P_polymer_root,

340 description='Polymer root directory (that contains the subdirectory for ancillary data)',

341 behavior=QgsProcessingParameterFile.Folder,

342 defaultValue=self._get_default_polymer_root(),

343 optional=True),

344 advanced=True)

345

346 self.addParameter(

347 QgsProcessingParameterNumber(

348 name=self.P_threads,

349 description='Number of threads for multiprocessing when running ACwater/Polymer \n'

350 "('0: no threads', '-1: automatic', '>0: number of threads')",

351 type=QgsProcessingParameterNumber.Integer,

352 defaultValue=-1, minValue=-1, maxValue=cpu_count()),

353 advanced=True)

354

355 self.addParameter(

356 QgsProcessingParameterNumber(

357 name=self.P_blocksize,

358 description='Block size for multiprocessing when running ACwater/Polymer',

359 type=QgsProcessingParameterNumber.Integer,

360 defaultValue=100, minValue=1),

361 advanced=True)

362

363 self.addParameter(

364 QgsProcessingParameterBoolean(

365 name=self.P_auto_download_ecmwf,

366 description='Automatically download ECMWF data for atmospheric correction '

367 'of water surfaces in ACwater/Polymer',

368 defaultValue=True),

369 advanced=True)

370

371 self.addParameter(

372 QgsProcessingParameterNumber(

373 name=self.P_scale_factor_boa_ref,

374 description='Scale factor to be applied to BOA reflectance result',

375 type=QgsProcessingParameterNumber.Integer,

376 defaultValue=10000, minValue=0),

377 advanced=True)

378

379 # self.addParameter(

380 # QgsProcessingParameterBoolean(

381 # name=self.P_run_smile_P,

382 # description='Smile detection and correction (provider smile coefficients are ignored)',

383 # defaultValue=False))

384

385 self.addParameter(

386 QgsProcessingParameterBoolean(

387 name=self.P_run_deadpix_P,

388 description='Dead pixel correction (based on L1B dead pixel map)',

389 defaultValue=False))

390

391 self.addParameter(

392 QgsProcessingParameterEnum(

393 name=self.P_deadpix_P_algorithm,

394 description="Algorithm for dead pixel correction",

395 options=['spectral', 'spatial'],

396 defaultValue=1),

397 advanced=True)

398

399 self.addParameter(

400 QgsProcessingParameterEnum(

401 name=self.P_deadpix_P_interp_spectral,

402 description="Spectral interpolation algorithm to be used during dead pixel correction ",

403 options=['linear', 'quadratic', 'cubic'],

404 defaultValue=0),

405 advanced=True)

406

407 self.addParameter(

408 QgsProcessingParameterEnum(

409 name=self.P_deadpix_P_interp_spatial,

410 description="Spatial interpolation algorithm to be used during dead pixel correction",

411 options=['linear', 'bilinear', 'cubic', 'spline'],

412 defaultValue=0),

413 advanced=True)

414

415 self.addParameter(

416 QgsProcessingParameterEnum(

417 name=self.P_ortho_resampAlg,

418 description="Ortho-rectification resampling algorithm",

419 options=['nearest', 'bilinear', 'gauss', 'cubic', 'cubic_spline', 'lanczos', 'average'],

420 defaultValue=1),

421 advanced=True)

422

423 self.addParameter(

424 QgsProcessingParameterEnum(

425 name=self.P_vswir_overlap_algorithm,

426 description="Algorithm specifying how to deal with the spectral bands "

427 "in the VNIR/SWIR spectral overlap region",

428 options=['VNIR and SWIR bands, order by wavelength', 'average VNIR and SWIR bands',

429 'VNIR bands only', 'SWIR bands only'],

430 defaultValue=3),

431 advanced=True)

432

433 self.addParameter(

434 QgsProcessingParameterEnum(

435 self.P_target_projection_type,

436 description='Projection type of the raster output files',

437 options=['UTM', 'Geographic'],

438 defaultValue=0),

439 advanced=True)

440

441 self.addParameter(

442 QgsProcessingParameterNumber(

443 name=self.P_target_epsg,

444 description='Custom EPSG code of the target projection (overrides target_projection_type)',

445 type=QgsProcessingParameterNumber.Integer,

446 defaultValue=None,

447 optional=True),

448 advanced=True)

449

450 # TODO:

451 # "target_coord_grid": "None" /*custom target coordinate grid to which the output is resampled

452 # ([x0, x1, y0, y1], e.g., [0, 30, 0, 30])*/

453

454 @staticmethod

455 def shortHelpString(*args, **kwargs):

456 """Display help string.

457

458 Example:

459 'Here comes the HTML documentation.' \

460 '<h3>With Headers...</h3>' \

461 'and Hyperlinks: <a href="www.google.de">Google</a>'

462

463 :param args:

464 :param kwargs:

465 """

466 text = \

467 'General information about this EnMAP box app can be found ' \

468 '<a href="https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/">here</a>. ' \

469 'For details, e.g., about all the algorithms implemented in EnPT, take a look at the ' \

470 '<a href="https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/index.html">EnPT backend ' \

471 'documentation</a>.' \

472 'Type enpt -h into a shell to get further information about individual parameters or check out ' \

473 'the <a href="https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/usage.html#' \

474 'command-line-utilities">documentation</a>.'

475

476 return text

477

478 def helpString(self):

479 return self.shortHelpString()

480

481 @staticmethod

482 def helpUrl(*args, **kwargs):

483 return 'https://enmap.git-pages.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp/doc/'

484

485 @staticmethod

486 def _get_preprocessed_parameters(parameters):

487 # replace Enum parameters with corresponding strings (not needed in case of unittest)

488 for n, opts in [

489 ('output_format', {0: 'GTiff', 1: 'ENVI'}),

490 ('output_interleave', {0: 'band', 1: 'line', 2: 'pixel'}),

491 ('mode_ac', {0: 'land', 1: 'water', 2: 'combined'}),

492 ('deadpix_P_algorithm', {0: 'spectral', 1: 'spatial'}),

493 ('deadpix_P_interp_spectral', {0: 'linear', 1: 'quadratic', 2: 'cubic'}),

494 ('deadpix_P_interp_spatial', {0: 'linear', 1: 'bilinear', 2: 'cubic', 3: 'spline'}),

495 ('ortho_resampAlg', {0: 'nearest', 1: 'bilinear', 2: 'gauss', 3: 'cubic',

496 4: 'cubic_spline', 5: 'lanczos', 6: 'average'}),

497 ('vswir_overlap_algorithm', {0: 'order_by_wvl', 1: 'average', 2: 'vnir_only', 3: 'swir_only'}),

498 ('target_projection_type', {0: 'UTM', 1: 'Geographic'}),

499 ]:

500 if isinstance(parameters[n], int):

501 parameters[n] = opts[parameters[n]]

502

503 # remove all parameters not to be forwarded to the EnPT CLI

504 parameters = {k: v for k, v in parameters.items()

505 if k not in ['conda_root']

506 and v not in [None, NULL, 'NULL', '']}

507

508 return parameters

509

510 @staticmethod

511 def _run_cmd(cmd, qgis_feedback=None, **kwargs):

512 """Execute external command and get its stdout, exitcode and stderr.

513

514 Code based on: https://stackoverflow.com/a/31867499

515

516 :param cmd: a normal shell command including parameters

517 """

518 def reader(pipe, queue):

519 try:

520 with pipe:

521 for line in iter(pipe.readline, b''):

522 queue.put((pipe, line))

523 finally:

524 queue.put(None)

525

526 process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, **kwargs)

527 q = Queue()

528 Thread(target=reader, args=[process.stdout, q]).start()

529 Thread(target=reader, args=[process.stderr, q]).start()

530

531 stdout_qname = None

532 stderr_qname = None

533

534 # for _ in range(2):

535 for source, line in iter(q.get, None):

536 if qgis_feedback.isCanceled():

537 # qgis_feedback.reportError('CANCELED')

538

539 proc2kill = psutil.Process(process.pid)

540 for proc in proc2kill.children(recursive=True):

541 proc.kill()

542 proc2kill.kill()

543

544 raise KeyboardInterrupt

545

546 linestr = line.decode('latin-1').rstrip()

547 # print("%s: %s" % (source, linestr))

548

549 # source name seems to be platfor/environment specific, so grab it from dummy STDOUT/STDERR messages.

550 if linestr == 'Connecting to EnPT STDOUT stream.':

551 stdout_qname = source.name

552 continue

553 if linestr == 'Connecting to EnPT STDERR stream.':

554 stderr_qname = source.name

555 continue

556

557 if source.name == stdout_qname:

558 qgis_feedback.pushInfo(linestr)

559 elif source.name == stderr_qname:

560 qgis_feedback.reportError(linestr)

561 else:

562 qgis_feedback.reportError(linestr)

563

564 exitcode = process.poll()

565

566 return exitcode

567

568 def _handle_results(self, parameters: dict, feedback, exitcode: int) -> dict:

569 success = False

570

571 if exitcode:

572 feedback.reportError("\n" +

573 "=" * 60 +

574 "\n" +

575 "An exception occurred. Processing failed.")

576

577 # list output dir

578 if 'output_dir' in parameters:

579 outdir = parameters['output_dir']

580 outraster_matches = \

581 glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.TIF')) or \

582 glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.bsq')) or \

583 glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.bil')) or \

584 glob(os.path.join(outdir, '*', '*SPECTRAL_IMAGE.bip'))

585 outraster = outraster_matches[0] if len(outraster_matches) > 0 else None

586

587 if os.path.isdir(outdir):

588 if os.listdir(outdir):

589 feedback.pushInfo("The output folder '%s' contains:\n" % outdir)

590 feedback.pushCommandInfo('\n'.join([os.path.basename(f) for f in os.listdir(outdir)]) + '\n')

591

592 if outraster:

593 subdir = os.path.dirname(outraster_matches[0])

594 feedback.pushInfo(subdir)

595 feedback.pushInfo("...where the folder '%s' contains:\n" % os.path.split(subdir)[-1])

596 feedback.pushCommandInfo('\n'.join(sorted([os.path.basename(f)

597 for f in os.listdir(subdir)])) + '\n')

598 success = True

599 else:

600 feedback.reportError("No output raster was written.")

601

602 else:

603 feedback.reportError("The output folder is empty.")

604

605 else:

606 feedback.reportError("No output folder created.")

607

608 # return outputs

609 if success:

610 return {

611 'success': True,

612 self.P_OUTPUT_RASTER: outraster,

613 # self.P_OUTPUT_VECTOR: parameters[self.P_OUTPUT_RASTER],

614 # self.P_OUTPUT_FILE: parameters[self.P_OUTPUT_RASTER],

615 self.P_OUTPUT_FOLDER: outdir

616 }

617 else:

618 return {'success': False}

619

620 else:

621 feedback.pushInfo('The output was skipped according to user setting.')

622 return {'success': True}

Coverage for enpt_enmapboxapp/_enpt_alg_base.py: 80%

198 statements