001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionListener; 009import java.io.BufferedReader; 010import java.io.IOException; 011import java.util.ArrayList; 012import java.util.Arrays; 013import java.util.Collection; 014import java.util.Collections; 015import java.util.HashMap; 016import java.util.HashSet; 017import java.util.List; 018import java.util.Locale; 019import java.util.Map; 020import java.util.Map.Entry; 021import java.util.Set; 022 023import javax.swing.JCheckBox; 024import javax.swing.JLabel; 025import javax.swing.JPanel; 026 027import org.openstreetmap.josm.command.ChangePropertyCommand; 028import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 029import org.openstreetmap.josm.command.Command; 030import org.openstreetmap.josm.command.SequenceCommand; 031import org.openstreetmap.josm.data.osm.AbstractPrimitive; 032import org.openstreetmap.josm.data.osm.OsmPrimitive; 033import org.openstreetmap.josm.data.osm.Tag; 034import org.openstreetmap.josm.data.osm.Tagged; 035import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 036import org.openstreetmap.josm.data.validation.Severity; 037import org.openstreetmap.josm.data.validation.Test.TagTest; 038import org.openstreetmap.josm.data.validation.TestError; 039import org.openstreetmap.josm.data.validation.util.Entities; 040import org.openstreetmap.josm.gui.progress.ProgressMonitor; 041import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 042import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 043import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 044import org.openstreetmap.josm.gui.tagging.presets.items.Check; 045import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 046import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 047import org.openstreetmap.josm.gui.widgets.EditableList; 048import org.openstreetmap.josm.io.CachedFile; 049import org.openstreetmap.josm.spi.preferences.Config; 050import org.openstreetmap.josm.tools.GBC; 051import org.openstreetmap.josm.tools.Logging; 052import org.openstreetmap.josm.tools.MultiMap; 053import org.openstreetmap.josm.tools.Utils; 054 055/** 056 * Check for misspelled or wrong tags 057 * 058 * @author frsantos 059 * @since 3669 060 */ 061public class TagChecker extends TagTest { 062 063 /** The config file of ignored tags */ 064 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 065 /** The config file of dictionary words */ 066 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 067 068 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 069 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 070 /** The spell check preset values which are not stored in TaggingPresets */ 071 private static volatile HashSet<String> additionalPresetsValueData; 072 /** often used tags which are not in presets */ 073 private static volatile MultiMap<String, String> oftenUsedTags = new MultiMap<>(); 074 075 /** The TagChecker data */ 076 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 077 private static final Set<String> ignoreDataEquals = new HashSet<>(); 078 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 079 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 080 /** tag keys that have only numerical values in the presets */ 081 private static final Set<String> ignoreForLevenshtein = new HashSet<>(); 082 083 /** The preferences prefix */ 084 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName(); 085 086 /** 087 * The preference key to check values 088 */ 089 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 090 /** 091 * The preference key to check keys 092 */ 093 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 094 /** 095 * The preference key to enable complex checks 096 */ 097 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 098 /** 099 * The preference key to search for fixme tags 100 */ 101 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 102 103 /** 104 * The preference key for source files 105 * @see #DEFAULT_SOURCES 106 */ 107 public static final String PREF_SOURCES = PREFIX + ".source"; 108 109 private static final String BEFORE_UPLOAD = "BeforeUpload"; 110 /** 111 * The preference key to check keys - used before upload 112 */ 113 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD; 114 /** 115 * The preference key to check values - used before upload 116 */ 117 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD; 118 /** 119 * The preference key to run complex tests - used before upload 120 */ 121 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD; 122 /** 123 * The preference key to search for fixmes - used before upload 124 */ 125 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD; 126 127 private static final int MAX_LEVENSHTEIN_DISTANCE = 2; 128 129 protected boolean checkKeys; 130 protected boolean checkValues; 131 /** Was used for special configuration file, might be used to disable value spell checker. */ 132 protected boolean checkComplex; 133 protected boolean checkFixmes; 134 135 protected JCheckBox prefCheckKeys; 136 protected JCheckBox prefCheckValues; 137 protected JCheckBox prefCheckComplex; 138 protected JCheckBox prefCheckFixmes; 139 protected JCheckBox prefCheckPaint; 140 141 protected JCheckBox prefCheckKeysBeforeUpload; 142 protected JCheckBox prefCheckValuesBeforeUpload; 143 protected JCheckBox prefCheckComplexBeforeUpload; 144 protected JCheckBox prefCheckFixmesBeforeUpload; 145 protected JCheckBox prefCheckPaintBeforeUpload; 146 147 // CHECKSTYLE.OFF: SingleSpaceSeparator 148 protected static final int EMPTY_VALUES = 1200; 149 protected static final int INVALID_KEY = 1201; 150 protected static final int INVALID_VALUE = 1202; 151 protected static final int FIXME = 1203; 152 protected static final int INVALID_SPACE = 1204; 153 protected static final int INVALID_KEY_SPACE = 1205; 154 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 155 protected static final int LONG_VALUE = 1208; 156 protected static final int LONG_KEY = 1209; 157 protected static final int LOW_CHAR_VALUE = 1210; 158 protected static final int LOW_CHAR_KEY = 1211; 159 protected static final int MISSPELLED_VALUE = 1212; 160 protected static final int MISSPELLED_KEY = 1213; 161 protected static final int MULTIPLE_SPACES = 1214; 162 protected static final int MISSPELLED_VALUE_NO_FIX = 1215; 163 // CHECKSTYLE.ON: SingleSpaceSeparator 164 165 protected EditableList sourcesList; 166 167 private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE); 168 169 /** 170 * Constructor 171 */ 172 public TagChecker() { 173 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 174 } 175 176 @Override 177 public void initialize() throws IOException { 178 initializeData(); 179 initializePresets(); 180 analysePresets(); 181 } 182 183 /** 184 * Add presets that contain only numerical values to the ignore list 185 */ 186 private static void analysePresets() { 187 for (String key : TaggingPresets.getPresetKeys()) { 188 if (isKeyIgnored(key)) 189 continue; 190 boolean allNumerical = true; 191 Set<String> values = TaggingPresets.getPresetValues(key); 192 if (values.isEmpty()) 193 allNumerical = false; 194 for (String val : values) { 195 if (!isNum(val)) { 196 allNumerical = false; 197 break; 198 } 199 } 200 if (allNumerical) { 201 ignoreForLevenshtein.add(key); 202 } 203 } 204 } 205 206 /** 207 * Reads the spell-check file into a HashMap. 208 * The data file is a list of words, beginning with +/-. If it starts with +, 209 * the word is valid, but if it starts with -, the word should be replaced 210 * by the nearest + word before this. 211 * 212 * @throws IOException if any I/O error occurs 213 */ 214 private static void initializeData() throws IOException { 215 ignoreDataStartsWith.clear(); 216 ignoreDataEquals.clear(); 217 ignoreDataEndsWith.clear(); 218 ignoreDataTag.clear(); 219 harmonizedKeys.clear(); 220 ignoreForLevenshtein.clear(); 221 oftenUsedTags.clear(); 222 223 StringBuilder errorSources = new StringBuilder(); 224 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) { 225 try ( 226 CachedFile cf = new CachedFile(source); 227 BufferedReader reader = cf.getContentReader() 228 ) { 229 String okValue = null; 230 boolean tagcheckerfile = false; 231 boolean ignorefile = false; 232 boolean isFirstLine = true; 233 String line; 234 while ((line = reader.readLine()) != null) { 235 if (line.isEmpty()) { 236 // ignore 237 } else if (line.startsWith("#")) { 238 if (line.startsWith("# JOSM TagChecker")) { 239 tagcheckerfile = true; 240 Logging.error(tr("Ignoring {0}. Support was dropped", source)); 241 } else 242 if (line.startsWith("# JOSM IgnoreTags")) { 243 ignorefile = true; 244 if (!DEFAULT_SOURCES.contains(source)) { 245 Logging.info(tr("Adding {0} to ignore tags", source)); 246 } 247 } 248 } else if (ignorefile) { 249 parseIgnoreFileLine(source, line); 250 } else if (tagcheckerfile) { 251 // ignore 252 } else if (line.charAt(0) == '+') { 253 okValue = line.substring(1); 254 } else if (line.charAt(0) == '-' && okValue != null) { 255 String hk = harmonizeKey(line.substring(1)); 256 if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null) { 257 Logging.debug(tr("Line was ignored: {0}", line)); 258 } 259 } else { 260 Logging.error(tr("Invalid spellcheck line: {0}", line)); 261 } 262 if (isFirstLine) { 263 isFirstLine = false; 264 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 265 Logging.info(tr("Adding {0} to spellchecker", source)); 266 } 267 } 268 } 269 } catch (IOException e) { 270 Logging.error(e); 271 errorSources.append(source).append('\n'); 272 } 273 } 274 275 if (errorSources.length() > 0) 276 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 277 } 278 279 /** 280 * Parse a line found in a configuration file 281 * @param source name of configuration file 282 * @param line the line to parse 283 */ 284 private static void parseIgnoreFileLine(String source, String line) { 285 line = line.trim(); 286 if (line.length() < 4) { 287 return; 288 } 289 try { 290 String key = line.substring(0, 2); 291 line = line.substring(2); 292 293 switch (key) { 294 case "S:": 295 ignoreDataStartsWith.add(line); 296 break; 297 case "E:": 298 ignoreDataEquals.add(line); 299 break; 300 case "F:": 301 ignoreDataEndsWith.add(line); 302 break; 303 case "K:": 304 Tag tag = Tag.ofString(line); 305 ignoreDataTag.add(tag); 306 oftenUsedTags.put(tag.getKey(), tag.getValue()); 307 break; 308 default: 309 if (!key.startsWith(";")) { 310 Logging.warn("Unsupported TagChecker key: " + key); 311 } 312 } 313 } catch (IllegalArgumentException e) { 314 Logging.error("Invalid line in {0} : {1}", source, e.getMessage()); 315 Logging.trace(e); 316 } 317 } 318 319 /** 320 * Reads the presets data. 321 * 322 */ 323 public static void initializePresets() { 324 325 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true)) 326 return; 327 328 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 329 if (!presets.isEmpty()) { 330 initAdditionalPresetsValueData(); 331 for (TaggingPreset p : presets) { 332 for (TaggingPresetItem i : p.data) { 333 if (i instanceof KeyedItem) { 334 addPresetValue((KeyedItem) i); 335 } else if (i instanceof CheckGroup) { 336 for (Check c : ((CheckGroup) i).checks) { 337 addPresetValue(c); 338 } 339 } 340 } 341 } 342 } 343 } 344 345 private static void initAdditionalPresetsValueData() { 346 additionalPresetsValueData = new HashSet<>(); 347 for (String a : AbstractPrimitive.getUninterestingKeys()) { 348 additionalPresetsValueData.add(a); 349 } 350 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys", 351 Arrays.asList("is_in", "int_ref", "fixme", "population"))) { 352 additionalPresetsValueData.add(a); 353 } 354 } 355 356 private static void addPresetValue(KeyedItem ky) { 357 if (ky.key != null && ky.getValues() != null) { 358 String hk = harmonizeKey(ky.key); 359 if (!ky.key.equals(hk)) { 360 harmonizedKeys.put(hk, ky.key); 361 } 362 } 363 } 364 365 /** 366 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 367 * @param s string to check 368 * @return {@code true} if {@code s} contains characters with code below 0x20 369 */ 370 private static boolean containsLow(String s) { 371 if (s == null) 372 return false; 373 for (int i = 0; i < s.length(); i++) { 374 if (s.charAt(i) < 0x20) 375 return true; 376 } 377 return false; 378 } 379 380 /** 381 * Get set of preset values for the given key. 382 * @param key the key 383 * @return null if key is not in presets or in additionalPresetsValueData, 384 * else a set which might be empty. 385 */ 386 private static Set<String> getPresetValues(String key) { 387 Set<String> res = TaggingPresets.getPresetValues(key); 388 if (res != null) 389 return res; 390 if (additionalPresetsValueData.contains(key)) 391 return Collections.emptySet(); 392 // null means key is not known 393 return null; 394 } 395 396 /** 397 * Determines if the given key is in internal presets. 398 * @param key key 399 * @return {@code true} if the given key is in internal presets 400 * @since 9023 401 */ 402 public static boolean isKeyInPresets(String key) { 403 return TaggingPresets.getPresetValues(key) != null; 404 } 405 406 /** 407 * Determines if the given tag is in internal presets. 408 * @param key key 409 * @param value value 410 * @return {@code true} if the given tag is in internal presets 411 * @since 9023 412 */ 413 public static boolean isTagInPresets(String key, String value) { 414 final Set<String> values = getPresetValues(key); 415 return values != null && values.contains(value); 416 } 417 418 /** 419 * Returns the list of ignored tags. 420 * @return the list of ignored tags 421 * @since 9023 422 */ 423 public static List<Tag> getIgnoredTags() { 424 return new ArrayList<>(ignoreDataTag); 425 } 426 427 /** 428 * Determines if the given tag key is ignored for checks "key/tag not in presets". 429 * @param key key 430 * @return true if the given key is ignored 431 */ 432 private static boolean isKeyIgnored(String key) { 433 if (ignoreDataEquals.contains(key)) { 434 return true; 435 } 436 for (String a : ignoreDataStartsWith) { 437 if (key.startsWith(a)) { 438 return true; 439 } 440 } 441 for (String a : ignoreDataEndsWith) { 442 if (key.endsWith(a)) { 443 return true; 444 } 445 } 446 return false; 447 } 448 449 /** 450 * Determines if the given tag is ignored for checks "key/tag not in presets". 451 * @param key key 452 * @param value value 453 * @return {@code true} if the given tag is ignored 454 * @since 9023 455 */ 456 public static boolean isTagIgnored(String key, String value) { 457 if (isKeyIgnored(key)) 458 return true; 459 final Set<String> values = getPresetValues(key); 460 if (values != null && values.isEmpty()) 461 return true; 462 if (!isTagInPresets(key, value)) { 463 for (Tag a : ignoreDataTag) { 464 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 465 return true; 466 } 467 } 468 } 469 return false; 470 } 471 472 /** 473 * Checks the primitive tags 474 * @param p The primitive to check 475 */ 476 @Override 477 public void check(OsmPrimitive p) { 478 if (!p.isTagged()) 479 return; 480 481 // Just a collection to know if a primitive has been already marked with error 482 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 483 484 for (Entry<String, String> prop : p.getKeys().entrySet()) { 485 String s = marktr("Tag ''{0}'' invalid."); 486 String key = prop.getKey(); 487 String value = prop.getValue(); 488 489 if (checkKeys) { 490 checkSingleTagKeySimple(withErrors, p, s, key); 491 } 492 if (checkValues) { 493 checkSingleTagValueSimple(withErrors, p, s, key, value); 494 checkSingleTagComplex(withErrors, p, key, value); 495 } 496 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) { 497 errors.add(TestError.builder(this, Severity.OTHER, FIXME) 498 .message(tr("FIXMES")) 499 .primitives(p) 500 .build()); 501 withErrors.put(p, "FIXME"); 502 } 503 } 504 } 505 506 private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) { 507 if (!checkValues || value == null) 508 return; 509 if ((containsLow(value)) && !withErrors.contains(p, "ICV")) { 510 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE) 511 .message(tr("Tag value contains character with code less than 0x20"), s, key) 512 .primitives(p) 513 .build()); 514 withErrors.put(p, "ICV"); 515 } 516 if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) { 517 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE) 518 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key) 519 .primitives(p) 520 .build()); 521 withErrors.put(p, "LV"); 522 } 523 if ((value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 524 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES) 525 .message(tr("Tags with empty values"), s, key) 526 .primitives(p) 527 .build()); 528 withErrors.put(p, "EV"); 529 } 530 final String errTypeSpace = "SPACE"; 531 if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) { 532 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE) 533 .message(tr("Property values start or end with white space"), s, key) 534 .primitives(p) 535 .build()); 536 withErrors.put(p, errTypeSpace); 537 } 538 if (value.contains(" ") && !withErrors.contains(p, errTypeSpace)) { 539 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES) 540 .message(tr("Property values contain multiple white spaces"), s, key) 541 .primitives(p) 542 .build()); 543 withErrors.put(p, errTypeSpace); 544 } 545 if (!value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 546 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML) 547 .message(tr("Property values contain HTML entity"), s, key) 548 .primitives(p) 549 .build()); 550 withErrors.put(p, "HTML"); 551 } 552 } 553 554 private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) { 555 if (!checkKeys || key == null) 556 return; 557 if ((containsLow(key)) && !withErrors.contains(p, "ICK")) { 558 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY) 559 .message(tr("Tag key contains character with code less than 0x20"), s, key) 560 .primitives(p) 561 .build()); 562 withErrors.put(p, "ICK"); 563 } 564 if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) { 565 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY) 566 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key) 567 .primitives(p) 568 .build()); 569 withErrors.put(p, "LK"); 570 } 571 if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 572 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE) 573 .message(tr("Invalid white space in property key"), s, key) 574 .primitives(p) 575 .build()); 576 withErrors.put(p, "IPK"); 577 } 578 } 579 580 private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) { 581 if (!checkValues || key == null || value == null || value.isEmpty()) 582 return; 583 if (additionalPresetsValueData != null && !isTagIgnored(key, value)) { 584 if (!isKeyInPresets(key)) { 585 spellCheckKey(withErrors, p, key); 586 } else if (!isTagInPresets(key, value)) { 587 if (oftenUsedTags.contains(key, value)) { 588 // tag is quite often used but not in presets 589 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 590 .message(tr("Presets do not contain property value"), 591 marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key) 592 .primitives(p) 593 .build()); 594 withErrors.put(p, "UPV"); 595 } else { 596 tryGuess(p, key, value, withErrors); 597 } 598 } 599 } 600 } 601 602 private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) { 603 String prettifiedKey = harmonizeKey(key); 604 String fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey); 605 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 606 // misspelled preset key 607 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY) 608 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey) 609 .primitives(p); 610 if (p.hasKey(fixedKey)) { 611 errors.add(error.build()); 612 } else { 613 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build()); 614 } 615 withErrors.put(p, "WPK"); 616 } else { 617 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 618 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key) 619 .primitives(p) 620 .build()); 621 withErrors.put(p, "UPK"); 622 } 623 } 624 625 private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) { 626 // try to fix common typos and check again if value is still unknown 627 final String harmonizedValue = harmonizeValue(value); 628 if (harmonizedValue == null || harmonizedValue.isEmpty()) 629 return; 630 String fixedValue = null; 631 List<Set<String>> sets = new ArrayList<>(); 632 Set<String> presetValues = getPresetValues(key); 633 if (presetValues != null) 634 sets.add(presetValues); 635 Set<String> usedValues = oftenUsedTags.get(key); 636 if (usedValues != null) 637 sets.add(usedValues); 638 for (Set<String> possibleValues: sets) { 639 if (possibleValues.contains(harmonizedValue)) { 640 fixedValue = harmonizedValue; 641 break; 642 } 643 } 644 if (fixedValue == null && !ignoreForLevenshtein.contains(key)) { 645 int maxPresetValueLen = 0; 646 List<String> fixVals = new ArrayList<>(); 647 // use Levenshtein distance to find typical typos 648 int minDist = MAX_LEVENSHTEIN_DISTANCE + 1; 649 String closest = null; 650 for (Set<String> possibleValues: sets) { 651 for (String possibleVal : possibleValues) { 652 if (possibleVal.isEmpty()) 653 continue; 654 maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length()); 655 if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) { 656 // don't suggest fix value when given value is short and lengths are too different 657 // for example surface=u would result in surface=mud 658 continue; 659 } 660 int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue); 661 if (dist >= harmonizedValue.length()) { 662 // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'. 663 continue; 664 } 665 if (dist < minDist) { 666 closest = possibleVal; 667 minDist = dist; 668 fixVals.clear(); 669 fixVals.add(possibleVal); 670 } else if (dist == minDist) { 671 fixVals.add(possibleVal); 672 } 673 } 674 } 675 676 if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE 677 && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) { 678 if (fixVals.size() < 2) { 679 fixedValue = closest; 680 } else { 681 Collections.sort(fixVals); 682 // misspelled preset value with multiple good alternatives 683 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX) 684 .message(tr("Unknown property value"), 685 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"), 686 value, key, fixVals) 687 .primitives(p).build()); 688 withErrors.put(p, "WPV"); 689 return; 690 } 691 } 692 } 693 if (fixedValue != null && !fixedValue.equals(value)) { 694 final String newValue = fixedValue; 695 // misspelled preset value 696 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE) 697 .message(tr("Unknown property value"), 698 marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue) 699 .primitives(p) 700 .build()); 701 withErrors.put(p, "WPV"); 702 } else { 703 // unknown preset value 704 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 705 .message(tr("Presets do not contain property value"), 706 marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key) 707 .primitives(p) 708 .build()); 709 withErrors.put(p, "UPV"); 710 } 711 } 712 713 private static boolean isNum(String harmonizedValue) { 714 try { 715 Double.parseDouble(harmonizedValue); 716 return true; 717 } catch (NumberFormatException e) { 718 return false; 719 } 720 } 721 722 private static boolean isFixme(String key, String value) { 723 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo") 724 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete"); 725 } 726 727 private static String harmonizeKey(String key) { 728 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,"); 729 } 730 731 private static String harmonizeValue(String value) { 732 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,"); 733 } 734 735 @Override 736 public void startTest(ProgressMonitor monitor) { 737 super.startTest(monitor); 738 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true); 739 if (isBeforeUpload) { 740 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 741 } 742 743 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true); 744 if (isBeforeUpload) { 745 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 746 } 747 748 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true); 749 if (isBeforeUpload) { 750 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 751 } 752 753 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true); 754 if (isBeforeUpload) { 755 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 756 } 757 } 758 759 @Override 760 public void visit(Collection<OsmPrimitive> selection) { 761 if (checkKeys || checkValues || checkComplex || checkFixmes) { 762 super.visit(selection); 763 } 764 } 765 766 @Override 767 public void addGui(JPanel testPanel) { 768 GBC a = GBC.eol(); 769 a.anchor = GridBagConstraints.EAST; 770 771 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 772 773 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true)); 774 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 775 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 776 777 prefCheckKeysBeforeUpload = new JCheckBox(); 778 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 779 testPanel.add(prefCheckKeysBeforeUpload, a); 780 781 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true)); 782 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 783 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 784 785 prefCheckComplexBeforeUpload = new JCheckBox(); 786 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 787 testPanel.add(prefCheckComplexBeforeUpload, a); 788 789 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES); 790 sourcesList = new EditableList(tr("TagChecker source")); 791 sourcesList.setItems(sources); 792 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 793 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 794 795 ActionListener disableCheckActionListener = e -> handlePrefEnable(); 796 prefCheckKeys.addActionListener(disableCheckActionListener); 797 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 798 prefCheckComplex.addActionListener(disableCheckActionListener); 799 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 800 801 handlePrefEnable(); 802 803 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true)); 804 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 805 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 806 807 prefCheckValuesBeforeUpload = new JCheckBox(); 808 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 809 testPanel.add(prefCheckValuesBeforeUpload, a); 810 811 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true)); 812 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 813 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 814 815 prefCheckFixmesBeforeUpload = new JCheckBox(); 816 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 817 testPanel.add(prefCheckFixmesBeforeUpload, a); 818 } 819 820 /** 821 * Enables/disables the source list field 822 */ 823 public void handlePrefEnable() { 824 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 825 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 826 sourcesList.setEnabled(selected); 827 } 828 829 @Override 830 public boolean ok() { 831 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 832 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 833 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 834 835 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 836 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 837 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 838 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 839 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 840 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 841 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 842 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 843 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems()); 844 } 845 846 @Override 847 public Command fixError(TestError testError) { 848 List<Command> commands = new ArrayList<>(50); 849 850 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 851 for (OsmPrimitive p : primitives) { 852 Map<String, String> tags = p.getKeys(); 853 if (tags.isEmpty()) { 854 continue; 855 } 856 857 for (Entry<String, String> prop: tags.entrySet()) { 858 String key = prop.getKey(); 859 String value = prop.getValue(); 860 if (value == null || value.trim().isEmpty()) { 861 commands.add(new ChangePropertyCommand(p, key, null)); 862 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 863 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value))); 864 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 865 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key))); 866 } else { 867 String evalue = Entities.unescape(value); 868 if (!evalue.equals(value)) { 869 commands.add(new ChangePropertyCommand(p, key, evalue)); 870 } 871 } 872 } 873 } 874 875 if (commands.isEmpty()) 876 return null; 877 if (commands.size() == 1) 878 return commands.get(0); 879 880 return new SequenceCommand(tr("Fix tags"), commands); 881 } 882 883 @Override 884 public boolean isFixable(TestError testError) { 885 if (testError.getTester() instanceof TagChecker) { 886 int code = testError.getCode(); 887 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 888 code == INVALID_KEY_SPACE || code == INVALID_HTML || 889 code == MULTIPLE_SPACES; 890 } 891 892 return false; 893 } 894}