Overhaul of message scoring & error correction.

Major changes:

Try to error-correct all messages that potentially could be DF11/17/18
even if the original DF value is different. This allows messages with
correctable damage in the first 5 bits to be correctable.

Track recently-seen DF18 addresses separately to DF17 addresses.
DF18 does not imply Mode S support, so we don't want to treat it like
a known Mode S emitter, but once we've heard some DF18 for an aircraft
we can be more confident about future DF18 messages for the same
address.

Rework the scoring system so it's just a big enum that lists all the
possible outcomes in the order that we want. This makes the relative
ordering of different messages clearer, and makes it easier to move
messages above/below the accept thresholds as needed.

Don't accept 2-bit-error-corrected messages that are from aircraft we
have not previously seen. This greatly reduces the number of garbage
messages when using 2-bit error correction.

Overall results are:

 * more CPU required for decoding (approx 30% increase in my tests)
   as we're doing a lot more speculative CRC-checking work
 * no significant change to message rates with error correction off
 * about 5% more 1-bit-corrected DF17 decodes, with a
   disproportionate increase in those messages contributing to
   successful position decodes / unique aircraft counts
 * _fewer_ decodes with 2-bit correction (versus old code with 2-bit
   correction), but the message quality is substantially improved,
   the rate of garbage decodes / phantom aircraft is greatly reduced

sample stats:

1-bit correction, old code:
    141158 total usable messages
    137852 accepted with correct CRC
      3306 accepted with 1-bit error repaired
     27446 DF17 messages
        51 unique aircraft tracks

1-bit correction, new code:
    141296 total usable messages
    137854 accepted with correct CRC
      3442 accepted with 1-bit error repaired
     27528 DF17 messages
        55 unique aircraft tracks

2-bit correction, old code:
    142656 total usable messages
    137809 accepted with correct CRC
      3283 accepted with 1-bit error repaired
      1564 accepted with 2-bit error repaired
     28803 DF17 messages
       349 unique aircraft tracks (<- note that most of these are garbage)

2-bit correction, new code:
    142426 total usable messages
    137822 accepted with correct CRC
      3420 accepted with 1-bit error repaired
      1184 accepted with 2-bit error repaired
     28666 DF17 messages
        55 unique aircraft tracks

crc.c

@@ -62,7 +62,7 @@ static void initLookupTables()
     }
 }
 
-uint32_t modesChecksum(uint8_t *message, int bits)
+uint32_t modesChecksum(const uint8_t *message, int bits)
 {
     uint32_t rem = 0;
     int i;
@@ -197,7 +197,7 @@ static struct errorinfo *prepareErrorTable(int bits, int max_correct, int max_de
 
     maxsize = 0;
     for (i = 1; i <= max_correct; ++i) {
-        maxsize += combinations(bits - 5, i); // space needed for all i-bit errors
+        maxsize += combinations(bits, i); // space needed for all i-bit errors
     }
 
 #ifdef CRCDEBUG
@@ -210,8 +210,7 @@ static struct errorinfo *prepareErrorTable(int bits, int max_correct, int max_de
     for (i = 0; i < MODES_MAX_BITERRORS; ++i)
         base_entry.bit[i] = -1;
 
-    // ignore the first 5 bits (DF type)
+    usedsize = prepareSubtable(table, 0, maxsize, 112 - bits, 0, bits, &base_entry, 0, max_correct);
-    usedsize = prepareSubtable(table, 0, maxsize, 112 - bits, 5, bits, &base_entry, 0, max_correct);
 
 #ifdef CRCDEBUG
     fprintf(stderr, "%d syndromes (expected %d).\n", usedsize, maxsize);
@@ -273,7 +272,7 @@ static struct errorinfo *prepareErrorTable(int bits, int max_correct, int max_de
         fprintf(stderr, "Flagging collisions between %d - %d bits..\n", max_correct+1, max_detect);
 #endif
 
-        flagged = flagCollisions(table, usedsize, 112 - bits, 5, bits, 0, 1, max_correct+1, max_detect);
+        flagged = flagCollisions(table, usedsize, 112 - bits, 0, bits, 0, 1, max_correct+1, max_detect);
 
 #ifdef CRCDEBUG
         fprintf(stderr, "Flagged %d collisions for removal.\n", flagged);
@@ -341,7 +340,7 @@ static struct errorinfo *prepareErrorTable(int bits, int max_correct, int max_de
                 if (table[j].errors == i)
                     ++count;
 
-            possible = combinations(bits-5, i);
+            possible = combinations(bits, i);
             fprintf(stderr, "  %d entries for %d-bit errors (%d possible, %d%% coverage)\n", count, i, possible, 100 * count / possible);
         }
 

crc.h

@@ -32,7 +32,7 @@ struct errorinfo {
 };
 
 void modesChecksumInit(int fixBits);
-uint32_t modesChecksum(uint8_t *msg, int bitlen);
+uint32_t modesChecksum(const uint8_t *msg, int bitlen);
 struct errorinfo *modesChecksumDiagnose(uint32_t syndrome, int bitlen);
 void modesChecksumFix(uint8_t *msg, struct errorinfo *info);
 

demod_2400.c

@@ -170,10 +170,10 @@ void demodulate2400(struct mag_buf *mag)
 
         // try all phases
         Modes.stats_current.demod_preambles++;
-        bestmsg = NULL; bestscore = -2; bestphase = -1;
+        bestmsg = NULL; bestscore = SR_NOT_SET; bestphase = -1;
         for (try_phase = 4; try_phase <= 8; ++try_phase) {
             uint16_t *pPtr;
-            int phase, i, score, bytelen;
+            int phase, i, score;
 
             // Decode all the next 112 bits, regardless of the actual message
             // size. We'll check the actual message type later
@@ -181,8 +181,7 @@ void demodulate2400(struct mag_buf *mag)
             pPtr = &m[j+19] + (try_phase/5);
             phase = try_phase % 5;
 
-            bytelen = MODES_LONG_MSG_BYTES;
+            for (i = 0; i < MODES_LONG_MSG_BYTES; ++i) {
-            for (i = 0; i < bytelen; ++i) {
                 uint8_t theByte = 0;
 
                 switch (phase) {
@@ -264,23 +263,10 @@ void demodulate2400(struct mag_buf *mag)
                 }
 
                 msg[i] = theByte;
-                if (i == 0) {
-                    switch (msg[0] >> 3) {
-                    case 0: case 4: case 5: case 11:
-                        bytelen = MODES_SHORT_MSG_BYTES; break;
-
-                    case 16: case 17: case 18: case 20: case 21: case 24:
-                        break;
-
-                    default:
-                        bytelen = 1; // unknown DF, give up immediately
-                        break;
-                    }
-                }
             }
 
             // Score the mode S message and see if it's any good.
-            score = scoreModesMessage(msg, i*8);
+            score = scoreModesMessage(msg);
             if (score > bestscore) {
                 // new high score!
                 bestmsg = msg;
@@ -295,8 +281,8 @@ void demodulate2400(struct mag_buf *mag)
         }
 
         // Do we have a candidate?
-        if (bestscore < 0) {
+        if (bestscore < SR_ACCEPT_THRESHOLD) {
-            if (bestscore == -1)
+            if (bestscore >= SR_UNKNOWN_THRESHOLD)
                 Modes.stats_current.demod_rejected_unknown_icao++;
             else
                 Modes.stats_current.demod_rejected_bad++;
@@ -319,17 +305,11 @@ void demodulate2400(struct mag_buf *mag)
         mm.score = bestscore;
 
         // Decode the received message
-        {
+        if (decodeModesMessage(&mm, bestmsg) < 0) {
-            int result = decodeModesMessage(&mm, bestmsg);
+            Modes.stats_current.demod_rejected_bad++;
-            if (result < 0) {
+            continue;
-                if (result == -1)
+        } else {
-                    Modes.stats_current.demod_rejected_unknown_icao++;
+            Modes.stats_current.demod_accepted[mm.correctedbits]++;
-                else
-                    Modes.stats_current.demod_rejected_bad++;
-                continue;
-            } else {
-                Modes.stats_current.demod_accepted[mm.correctedbits]++;
-            }
         }
 
         // measure signal power

icao_filter.c

@@ -81,18 +81,6 @@ void icaoFilterAdd(uint32_t addr)
     }
     if (icao_filter_active[h] == EMPTY)
         icao_filter_active[h] = addr;
-
-    // also add with a zeroed top byte, for handling DF20/21 with Data Parity
-    h0 = h = icaoHash(addr & 0x00ffff);
-    while (icao_filter_active[h] != EMPTY && (icao_filter_active[h] & 0x00ffff) != (addr & 0x00ffff)) {
-        h = (h+1) & (ICAO_FILTER_SIZE-1);
-        if (h == h0) {
-            fprintf(stderr, "ICAO hash table full, increase ICAO_FILTER_SIZE\n");
-            return;
-        }
-    }
-    if (icao_filter_active[h] == EMPTY)
-        icao_filter_active[h] = addr;
 }
 
 int icaoFilterTest(uint32_t addr)
@@ -120,32 +108,6 @@ int icaoFilterTest(uint32_t addr)
     return 0;
 }
 
-uint32_t icaoFilterTestFuzzy(uint32_t partial)
-{
-    uint32_t h, h0;
-
-    partial &= 0x00ffff;
-    h0 = h = icaoHash(partial);
-    while (icao_filter_a[h] != EMPTY && (icao_filter_a[h] & 0x00ffff) != partial) {
-        h = (h+1) & (ICAO_FILTER_SIZE-1);
-        if (h == h0)
-            break;
-    }
-    if (icao_filter_a[h] != EMPTY && (icao_filter_a[h] & 0x00ffff) == partial)
-        return icao_filter_a[h];
-
-    h = h0;
-    while (icao_filter_b[h] != EMPTY && (icao_filter_b[h] & 0x00ffff) != partial) {
-        h = (h+1) & (ICAO_FILTER_SIZE-1);
-        if (h == h0)
-            break;
-    }
-    if (icao_filter_b[h] != EMPTY && (icao_filter_b[h] & 0x00ffff) == partial)
-        return icao_filter_b[h];
-
-    return 0;
-}
-
 // call this periodically:
 void icaoFilterExpire()
 {

icao_filter.h

@@ -20,6 +20,9 @@
 #ifndef DUMP1090_ICAO_FILTER_H
 #define DUMP1090_ICAO_FILTER_H
 
+// Special address bit used to mark ADS-B (NT) emitters
+#define ICAO_FILTER_ADSB_NT (1 << 25)
+
 // Call once:
 void icaoFilterInit();
 

mode_s.c

@@ -3,6 +3,7 @@
 // mode_s.c: Mode S message decoding.
 //
 // Copyright (c) 2014-2016 Oliver Jowett <oliver@mutability.co.uk>
+// Copyright (c) 2021 FlightAware LLC
 //
 // This file is free software: you may copy, redistribute and/or modify it
 // under the terms of the GNU General Public License as published by the
@@ -224,77 +225,143 @@ static float decodeMovementFieldV0(unsigned movement) {
     else                       return 0;
 }
 
-// Correct a decoded native-endian Address Announced field
+// Apply possible corrections to the 14-byte message in "in", storing the result in "out"
-// (from bits 8-31) if it is affected by the given error
+//
-// syndrome. Updates *addr and returns >0 if changed, 0 if
+// If the message has a correct CRC, copies in to out unchanged and returns 0
-// it was unaffected.
+// If the message has correctable errors, applies the corrections to out and returns the number of corrected errors
-static int correct_aa_field(uint32_t *addr, struct errorinfo *ei)
+// If the message is uncorrectable (this may mean the message type does not have CRC coverage), returns -1
+
+// is this message a long-form message with a DF that uses Parity/Interrogator?
+static bool isLongPIMessage(const unsigned char *msg)
 {
-    int i;
+    const unsigned df = getbits(msg, 1, 5);
-    int addr_errors = 0;
+    if (df == 17 || df == 18)
+        return true;
+    return false;
+}
 
-    if (!ei)
+// is this message a short-form message with a DF that uses Parity/Interrogator?
-        return 0;
+static bool isShortPIMessage(const unsigned char *msg)
+{
+    const unsigned df = getbits(msg, 1, 5);
+    return (df == 11); // assume IID==0
+}
 
-    for (i = 0; i < ei->errors; ++i) {
+static int correctMessage(const unsigned char *in, unsigned char *out)
-        if (ei->bit[i] >= 8 && ei->bit[i] <= 31) {
+{
-            *addr ^= 1 << (31 - ei->bit[i]);
+    // Possible DF values of the first byte of a message that could be a valid DF11/17/18
-            ++addr_errors;
+    // message after correction. See tools/df-correction-arrays.py for generator code.
+    // This is used to shortcut message correction so that we don't bother computing a CRC over
+    // messages that couldn't possibly become one of those message types.
+
+    // These are bitsets, where the bit with value 1<<N represents a match for DF N
+    static const uint32_t df_correctable_short[MODES_MAX_BITERRORS + 1] = {
+        0x00000800, 0x08008e08, 0x08008e08
+    };
+    static const uint32_t df_correctable_long[MODES_MAX_BITERRORS + 1] = {
+        0x00060000, 0x066f0006, 0x6fff066f
+    };
+
+    // Try to correct, including corrections to the initial 5 bit DF field
+    // that determines message format
+
+    const unsigned uncorrected_df = getbits(in, 1, 5);
+    const uint32_t df_bit = 1 << uncorrected_df;
+
+    struct errorinfo *long_ei = NULL;
+    if (df_correctable_long[Modes.nfix_crc] & df_bit) {
+        uint32_t long_syndrome = modesChecksum(in, MODES_LONG_MSG_BITS);
+        if (isLongPIMessage(in) && long_syndrome == 0) {
+            // DF17/18 message with correct checksum
+            memcpy(out, in, MODES_LONG_MSG_BYTES);
+            return 0;
+        }
+
+        long_ei = modesChecksumDiagnose(long_syndrome, MODES_LONG_MSG_BITS);
+    }
+
+    struct errorinfo *short_ei = NULL;
+    if (df_correctable_short[Modes.nfix_crc] & df_bit) {
+        uint32_t short_syndrome = modesChecksum(in, MODES_SHORT_MSG_BITS);
+        if (isShortPIMessage(in) && (short_syndrome & 0xFFFF80) == 0) {
+            // DF11 message with correct checksum
+            // (low 7 bits may be IID)
+            memcpy(out, in, MODES_SHORT_MSG_BYTES);
+            return 0;
+        }
+
+        short_ei = modesChecksumDiagnose(short_syndrome, MODES_SHORT_MSG_BITS); // assume IID == 0
+    }
+
+    // Might be a damaged DF11/17/18, or might be another message type that doesn't have a full CRC
+
+    unsigned long_errors = (long_ei ? long_ei->errors : 999);
+    unsigned short_errors = (short_ei ? short_ei->errors : 999);
+
+    // If both 56-bit and 112-bit corrections are possible:
+    //   try the correction with fewer error bits first
+    //   if there's a tie, try the 112-bit version first
+
+    if (long_ei && long_errors <= short_errors) {
+        memcpy(out, in, MODES_LONG_MSG_BYTES);
+        modesChecksumFix(out, long_ei);
+        if (isLongPIMessage(out)) {
+            // valid DF17/18 message after corrections
+            return long_errors;
         }
     }
 
-    return addr_errors;
+    // Don't try to correct >1 error in DF11 (see crc.c)
+    if (short_ei && short_errors == 1) {
+        memcpy(out, in, MODES_SHORT_MSG_BYTES);
+        modesChecksumFix(out, short_ei);
+        if (isShortPIMessage(out)) {
+            // valid DF11 message after corrections
+            return short_errors;
+        }
+    }
+
+    if (long_ei && long_errors > short_errors) {
+        memcpy(out, in, MODES_LONG_MSG_BYTES);
+        modesChecksumFix(out, long_ei);
+        if (isLongPIMessage(out)) {
+            // valid DF17/18 message after corrections
+            return long_errors;
+        }
+    }
+
+    // Nothing more to try, we can't correct this one further
+    memcpy(out, in, MODES_LONG_MSG_BYTES);
+    return -1;
 }
 
 // Score how plausible this ModeS message looks.
-// The more positive, the more reliable the message is
+// The more positive, the more reliable the message is.
-
+score_rank scoreModesMessage(const unsigned char *uncorrected)
-// 1000: DF 0/4/5/16/24 with a CRC-derived address matching a known aircraft
-
-// 1800: DF17/18 with good CRC and an address matching a known aircraft
-// 1400: DF17/18 with good CRC and an address not matching a known aircraft
-//  900: DF17/18 with 1-bit error and an address matching a known aircraft
-//  700: DF17/18 with 1-bit error and an address not matching a known aircraft
-//  450: DF17/18 with 2-bit error and an address matching a known aircraft
-//  350: DF17/18 with 2-bit error and an address not matching a known aircraft
-
-// 1600: DF11 with IID==0, good CRC and an address matching a known aircraft
-// 1000: DF11 with IID!=0, good CRC and an address matching a known aircraft
-//  800: DF11 with 1-bit error and an address matching a known aircraft
-//  750: DF11 with IID==0, good CRC and an address not matching a known aircraft
-
-// 1000: DF20/21 with a CRC-derived address matching a known aircraft
-//  500: DF20/21 with a CRC-derived address matching a known aircraft (bottom 16 bits only - overlay control in use)
-
-//   -1: message might be valid, but we couldn't validate the CRC against a known ICAO
-//   -2: bad message or unrepairable CRC error
-
-static unsigned char all_zeros[14] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-int scoreModesMessage(unsigned char *msg, int validbits)
 {
-    int msgtype, msgbits, crc, iid;
+    // try to produce a corrected DF11/17/18, including correcting the DF bits
-    uint32_t addr;
+    unsigned char corrected[14];
-    struct errorinfo *ei;
+    int corrections = correctMessage(uncorrected, corrected);
 
-    if (validbits < 56)
+    // This is a "valid" DF0 message, but it's not useful; we discard these messages
-        return -2;
+    static const unsigned char all_zeros[MODES_SHORT_MSG_BYTES] = { 0, 0, 0, 0, 0, 0, 0 };
+    if (!memcmp(all_zeros, corrected, sizeof(all_zeros)))
+        return SR_ALL_ZEROS;
 
-    msgtype = getbits(msg, 1, 5); // Downlink Format
+    unsigned df = getbits(corrected, 1, 5); // Downlink Format
-    msgbits = modesMessageLenByType(msgtype);
+    switch (df) {
+    case 0:  // short air-air surveillance
+    case 4:  // surveillance, altitude reply
+    case 5:  // surveillance, altitude reply
+        {
+            uint32_t addr = modesChecksum(corrected, MODES_SHORT_MSG_BITS);
+            bool recent = icaoFilterTest(addr);
+            return recent ? SR_UNRELIABLE_KNOWN : SR_UNRELIABLE_UNKNOWN;
+        }
 
-    if (validbits < msgbits)
-        return -2;
-
-    if (!memcmp(all_zeros, msg, msgbits/8))
-        return -2;
-
-    crc = modesChecksum(msg, msgbits);
-
-    switch (msgtype) {
-    case 0: // short air-air surveillance
-    case 4: // surveillance, altitude reply
-    case 5: // surveillance, altitude reply
     case 16: // long air-air surveillance
+    case 20: // Comm-B, altitude reply
+    case 21: // Comm-B, identity reply
     case 24: // Comm-D (ELM)
     case 25: // Comm-D (ELM)
     case 26: // Comm-D (ELM)
@@ -303,137 +370,165 @@ int scoreModesMessage(unsigned char *msg, int validbits)
     case 29: // Comm-D (ELM)
     case 30: // Comm-D (ELM)
     case 31: // Comm-D (ELM)
-        return icaoFilterTest(crc) ? 1000 : -1;
+        {
-
+            uint32_t addr = modesChecksum(corrected, MODES_LONG_MSG_BITS);
-    case 11: // All-call reply
+            bool recent = icaoFilterTest(addr);
-        iid = crc & 0x7f;
+            return recent ? SR_UNRELIABLE_KNOWN : SR_UNRELIABLE_UNKNOWN;
-        addr = getbits(msg, 9, 32);
-
-        if (crc & 0xffff80) {
-            // Try to diagnose based on the _full_ CRC
-            // i.e. under the assumption that IID = 0
-            ei = modesChecksumDiagnose(crc, msgbits);
-            if (!ei)
-                return -2; // can't correct errors
-
-            // see crc.c comments: we do not attempt to fix
-            // more than single-bit errors, as two-bit
-            // errors are ambiguous in DF11.
-            if (ei->errors > 1)
-                return -2; // can't correct errors
-
-            // fix any errors in the address field
-            correct_aa_field(&addr, ei);
-
-            // here, IID = 0 implicitly
-            if (icaoFilterTest(addr))
-                return 800;
-            else
-                return -1;
         }
 
-        // CRC was correct (ish)
+    case 11:
-        if (iid == 0) {
+        {
-            if (icaoFilterTest(addr))
+            // DF11 All-call reply
-                return 1600;
+            uint32_t addr = getbits(corrected, 9, 32);
-            else
+            uint32_t iid = modesChecksum(corrected, MODES_SHORT_MSG_BITS) & 0x7F;
-                return 750;
+            bool recent = icaoFilterTest(addr);
-        } else { // iid != 0
+
-            if (icaoFilterTest(addr))
+            switch (corrections) {
-                return 1000;
+            case 0:
-            else
+                if (iid == 0)
-                return -1;
+                    return recent ? SR_DF11_ACQ_KNOWN : SR_DF11_ACQ_UNKNOWN;
+                else
+                    return recent ? SR_DF11_IID_KNOWN : SR_DF11_IID_UNKNOWN;
+            case 1:
+                if (iid == 0)
+                    return recent ? SR_DF11_ACQ_1ERROR_KNOWN : SR_DF11_ACQ_1ERROR_UNKNOWN;
+                else
+                    return recent ? SR_DF11_IID_1ERROR_KNOWN : SR_DF11_IID_1ERROR_UNKNOWN;
+            default:
+                return SR_UNCORRECTABLE;
+            }
         }
 
     case 17:   // Extended squitter
+        {
+            uint32_t addr = getbits(corrected, 9, 32);
+            bool recent = icaoFilterTest(addr);
+
+            switch (corrections) {
+            case 0:
+                return recent ? SR_DF17_KNOWN : SR_DF17_UNKNOWN;
+            case 1:
+                return recent ? SR_DF17_1ERROR_KNOWN : SR_DF17_1ERROR_UNKNOWN;
+            case 2:
+                return recent ? SR_DF17_2ERROR_KNOWN : SR_DF17_2ERROR_UNKNOWN;
+            default:
+                return SR_UNCORRECTABLE;
+            }
+        }
+
     case 18:   // Extended squitter/non-transponder
-        ei = modesChecksumDiagnose(crc, msgbits);
+        {
-        if (!ei)
+            uint32_t addr = getbits(corrected, 9, 32);
-            return -2; // can't correct errors
+            bool recent = icaoFilterTest(addr | ICAO_FILTER_ADSB_NT); // only look for previous DF18 activity
 
-        // fix any errors in the address field
+            switch (corrections) {
-        addr = getbits(msg, 9, 32);
+            case 0:
-        correct_aa_field(&addr, ei);
+                return recent ? SR_DF18_KNOWN : SR_DF18_UNKNOWN;
+            case 1:
+                return recent ? SR_DF18_1ERROR_KNOWN : SR_DF18_1ERROR_UNKNOWN;
+            case 2:
+                return recent ? SR_DF18_2ERROR_KNOWN : SR_DF18_2ERROR_UNKNOWN;
+            default:
+                return SR_UNCORRECTABLE;
+            }
+        }
 
-        if (icaoFilterTest(addr))
-            return 1800 / (ei->errors+1);
-        else
-            return 1400 / (ei->errors+1);
 
-    case 20:   // Comm-B, altitude reply
-    case 21:   // Comm-B, identity reply
-        if (icaoFilterTest(crc))
-            return 1000; // Address/Parity
-
-#if 0
-        // This doesn't seem useful, as we mistake a lot of CRC errors
-        // for overlay control
-        if (icaoFilterTestFuzzy(crc))
-            return 500;  // Data/Parity
-#endif
-
-        return -2;
 
     default:
         // unknown message type
-        return -2;
+        return SR_UNKNOWN_DF;
     }
 }
 
+static const char *score_to_string(score_rank score)
+{
+    switch (score) {
+    case SR_NOT_SET: return "NOT_SET";
+    case SR_UNKNOWN_THRESHOLD: return "UNKNOWN_THRESHOLD";
+    case SR_ACCEPT_THRESHOLD: return "ACCEPT_THRESHOLD";
+
+    case SR_ALL_ZEROS: return "ALL_ZEROS";
+    case SR_UNKNOWN_DF: return "UNKNOWN_DF";
+    case SR_UNCORRECTABLE: return "UNCORRECTABLE";
+
+    case SR_UNRELIABLE_UNKNOWN: return "UNRELIABLE_UNKNOWN";
+    case SR_UNRELIABLE_KNOWN: return "UNRELIABLE_KNOWN";
+
+    case SR_DF11_IID_1ERROR_UNKNOWN: return "DF11_IID_1ERROR_UNKNOWN";
+    case SR_DF11_ACQ_1ERROR_UNKNOWN: return "DF11_ACQ_1ERROR_UNKNOWN";
+    case SR_DF11_IID_UNKNOWN: return "DF11_IID_UNKNOWN";
+    case SR_DF11_ACQ_UNKNOWN: return "DF11_ACQ_UNKNOWN";
+    case SR_DF11_IID_1ERROR_KNOWN: return "DF11_IID_1ERROR_KNOWN";
+    case SR_DF11_ACQ_1ERROR_KNOWN: return "DF11_ACQ_1ERROR_KNOWN";
+    case SR_DF11_IID_KNOWN: return "DF11_IID_KNOWN";
+    case SR_DF11_ACQ_KNOWN: return "DF11_ACQ_KNOWN";
+
+    case SR_DF17_2ERROR_UNKNOWN: return "DF17_2ERROR_UNKNOWN";
+    case SR_DF17_2ERROR_KNOWN: return "DF17_2ERROR_KNOWN";
+    case SR_DF17_1ERROR_UNKNOWN: return "DF17_1ERROR_UNKNOWN";
+    case SR_DF17_1ERROR_KNOWN: return "DF17_1ERROR_KNOWN";
+    case SR_DF17_UNKNOWN: return "DF17_UNKNOWN";
+    case SR_DF17_KNOWN: return "DF17_KNOWN";
+
+    case SR_DF18_2ERROR_UNKNOWN: return "DF18_2ERROR_UNKNOWN";
+    case SR_DF18_2ERROR_KNOWN: return "DF18_2ERROR_KNOWN";
+    case SR_DF18_1ERROR_UNKNOWN: return "DF18_1ERROR_UNKNOWN";
+    case SR_DF18_1ERROR_KNOWN: return "DF18_1ERROR_KNOWN";
+    case SR_DF18_UNKNOWN: return "DF18_UNKNOWN";
+    case SR_DF18_KNOWN: return "DF18_KNOWN";
+    }
+
+    return "<bad value>";
+}
+
+static void decodeExtendedSquitter(struct modesMessage *mm);
+
 //
 //=========================================================================
 //
 // Decode a raw Mode S message demodulated as a stream of bytes by detectModeS(),
 // and split it into fields populating a modesMessage structure.
 //
-
-static void decodeExtendedSquitter(struct modesMessage *mm);
-
 // return 0 if all OK
-//   -1: message might be valid, but we couldn't validate the CRC against a known ICAO
+// <0 if it's a bad message
-//   -2: bad message or unrepairable CRC error
+//
-
+int decodeModesMessage(struct modesMessage *mm, const unsigned char *in)
-int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
 {
-    // Preserve the original uncorrected copy for later forwarding
+    // score the message if needed (it might be coming off the network)
-    memcpy(mm->verbatim, msg, MODES_LONG_MSG_BYTES);
+    if (mm->score == SR_NOT_SET)
-    // Work on our local copy.
+        mm->score = scoreModesMessage(in);
-    memcpy(mm->msg, msg, MODES_LONG_MSG_BYTES);
-    msg = mm->msg;
 
-    // don't accept all-zeros messages
+    if (mm->score < SR_UNKNOWN_THRESHOLD)
-    if (!memcmp(all_zeros, msg, 7))
+        return -1;
+    if (mm->score < SR_ACCEPT_THRESHOLD)
         return -2;
 
+    // Preserve the original uncorrected copy for later forwarding
+    memcpy(mm->verbatim, in, MODES_LONG_MSG_BYTES);
+
+    // Apply corrections to our local copy
+    int corrections = correctMessage(in, mm->msg);
+    const unsigned char *msg = mm->msg;
+
     // Get the message type ASAP as other operations depend on this
     mm->msgtype         = getbits(msg, 1, 5); // Downlink Format
     mm->msgbits         = modesMessageLenByType(mm->msgtype);
     mm->crc             = modesChecksum(msg, mm->msgbits);
-    mm->correctedbits   = 0;
+    mm->correctedbits   = corrections > 0 ? corrections : 0;
     mm->addr            = 0;
 
     // Do checksum work and set fields that depend on the CRC
     switch (mm->msgtype) {
     case 0: // short air-air surveillance
     case 4: // surveillance, altitude reply
-    case 5: // surveillance, altitude reply
+    case 5: // surveillance, identity reply
     case 16: // long air-air surveillance
-    case 24: // Comm-D (ELM)
+        // These message types use Address/Parity
-    case 25: // Comm-D (ELM)
+        // so we can't check the CRC and must infer the transmitter's address
-    case 26: // Comm-D (ELM)
-    case 27: // Comm-D (ELM)
-    case 28: // Comm-D (ELM)
-    case 29: // Comm-D (ELM)
-    case 30: // Comm-D (ELM)
-    case 31: // Comm-D (ELM)
-        // These message types use Address/Parity, i.e. our CRC syndrome is the sender's ICAO address.
-        // We can't tell if the CRC is correct or not as we don't know the correct address.
-        // Accept the message if it appears to be from a previously-seen aircraft
-        if (!icaoFilterTest(mm->crc)) {
-           return -1;
-        }
         mm->source = SOURCE_MODE_S;
         mm->addr = mm->crc;
+        mm->reliable = 0;
         break;
 
     case 11: // All-call reply
@@ -442,65 +537,14 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
         //
         // however! CL + IC only occupy the lower 7 bits of the CRC. So if we ignore those bits when testing
         // the CRC we can still try to detect/correct errors.
-
         mm->IID = mm->crc & 0x7f;
-        if (mm->crc & 0xffff80) {
-            // Try to diagnose based on the _full_ CRC
-            // i.e. under the assumption that IID = 0
-
-            int addr;
-            struct errorinfo *ei = modesChecksumDiagnose(mm->crc, mm->msgbits);
-            if (!ei) {
-                return -2; // couldn't fix it
-            }
-
-            // see crc.c comments: we do not attempt to fix
-            // more than single-bit errors, as two-bit
-            // errors are ambiguous in DF11.
-            if (ei->errors > 1)
-                return -2; // can't correct errors
-
-            mm->correctedbits = ei->errors;
-            mm->IID = 0;
-            modesChecksumFix(msg, ei);
-
-            // check whether the corrected message looks sensible
-            // we are conservative here: only accept corrected messages that
-            // match an existing aircraft.
-            addr = getbits(msg, 9, 32);
-            if (!icaoFilterTest(addr)) {
-                return -1;
-            }
-        }
         mm->source = SOURCE_MODE_S_CHECKED;
         mm->reliable = (mm->IID == 0 && mm->correctedbits == 0);
         break;
 
     case 17:   // Extended squitter
     case 18: { // Extended squitter/non-transponder
-        struct errorinfo *ei;
-        int addr1, addr2;
-
         // These message types use Parity/Interrogator, but are specified to set II=0
-
-        if (mm->crc != 0) {
-            ei = modesChecksumDiagnose(mm->crc, mm->msgbits);
-            if (!ei) {
-                return -2; // couldn't fix it
-            }
-
-            addr1 = getbits(msg, 9, 32);
-            mm->correctedbits = ei->errors;
-            modesChecksumFix(msg, ei);
-            addr2 = getbits(msg, 9, 32);
-
-            // we are conservative here: only accept corrected messages that
-            // match an existing aircraft.
-            if (addr1 != addr2 && !icaoFilterTest(addr2)) {
-                return -1;
-            }
-        }
-
         mm->source = SOURCE_ADSB; // TIS-B decoding will override this if needed
         mm->reliable = (mm->correctedbits == 0);
         break;
@@ -508,22 +552,32 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
 
     case 20: // Comm-B, altitude reply
     case 21: // Comm-B, identity reply
-        // These message types either use Address/Parity (see DF0 etc)
+        // These message types either use Address/Parity
         // or Data Parity where the requested BDS is also xored into the top byte.
         // So not only do we not know whether the CRC is right, we also don't know if
         // the ICAO is right! Ow.
 
-        // Try an exact match
+        mm->source = SOURCE_MODE_S;
-        if (icaoFilterTest(mm->crc)) {
+        mm->addr = mm->crc;
-            // OK.
+        mm->reliable = 0;
-            mm->source = SOURCE_MODE_S;
+        break;
-            mm->addr = mm->crc;
-            break;
-        }
 
-        // BDS / overlay control just doesn't work out.
+    case 24: // Comm-D (ELM)
-
+    case 25: // Comm-D (ELM)
-        return -1; // no good
+    case 26: // Comm-D (ELM)
+    case 27: // Comm-D (ELM)
+    case 28: // Comm-D (ELM)
+    case 29: // Comm-D (ELM)
+    case 30: // Comm-D (ELM)
+    case 31: // Comm-D (ELM)
+        // These messages use Address/Parity,
+        // and also use some of the DF bits to carry data. Remap them all to a single
+        // DF for simplicity.
+        mm->msgtype = 24;
+        mm->source = SOURCE_MODE_S;
+        mm->addr = mm->crc;
+        mm->reliable = 0;
+        break;
 
     default:
         // All other message types, we don't know how to handle their CRCs, give up
@@ -635,7 +689,7 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
     }
 
     // KE (Control, ELM)
-    if (mm->msgtype >= 24 && mm->msgtype <= 31) {
+    if (mm->msgtype == 24) {
         mm->KE = getbit(msg, 4);
     }
 
@@ -646,7 +700,7 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
     }
 
     // MD (message, Comm-D)
-    if (mm->msgtype >= 24 && mm->msgtype <= 31) {
+    if (mm->msgtype == 24) {
         memcpy(mm->MD, &msg[1], 10);
     }
 
@@ -662,7 +716,7 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
     }
 
     // ND (number of D-segment, Comm-D)
-    if (mm->msgtype >= 24 && mm->msgtype <= 31) {
+    if (mm->msgtype == 24) {
         mm->ND = getbits(msg, 5, 8);
     }
 
@@ -691,15 +745,13 @@ int decodeModesMessage(struct modesMessage *mm, unsigned char *msg)
     }
 
     if (!mm->correctedbits && (mm->msgtype == 17 || (mm->msgtype == 11 && mm->IID == 0))) {
-        // No CRC errors seen, and either it was an DF17 extended squitter
+        // DF17 ADS-B or DF11 acquisition squitter. Mark as known Mode-S source
-        // or a DF11 acquisition squitter with II = 0. We probably have the right address.
-
-        // Don't do this for DF18, as a DF18 transmitter doesn't necessarily have a
-        // Mode S transponder.
-
-        // NB this is the only place that adds addresses!
         icaoFilterAdd(mm->addr);
     }
+    if (!mm->correctedbits && mm->msgtype == 18) {
+         // Mark as known ADS-B (NT) source
+       icaoFilterAdd(mm->addr | ICAO_FILTER_ADSB_NT);
+    }
 
     // MLAT overrides all other sources
     if (mm->remote && mm->timestampMsg == MAGIC_MLAT_TIMESTAMP)
@@ -1798,7 +1850,7 @@ void displayModesMessage(struct modesMessage *mm) {
         printf("RSSI: %.1f dBFS\n", 10 * log10(mm->signalLevel));
 
     if (mm->score)
-        printf("Score: %d\n", mm->score);
+        printf("Score: %d (%s)\n", mm->score, score_to_string(mm->score));
 
     if (mm->timestampMsg) {
         if (mm->timestampMsg == MAGIC_MLAT_TIMESTAMP)
@@ -1864,18 +1916,16 @@ void displayModesMessage(struct modesMessage *mm) {
         break;
 
     case 24:
-    case 25:
+        /* 25 .. 31 also remapped to 24 during decoding */
-    case 26:
-    case 27:
-    case 28:
-    case 29:
-    case 30:
-    case 31:
         printf("DF:24 addr:%06x KE:%u ND:%u MD:",
                mm->addr, mm->KE, mm->ND);
         print_hex_bytes(mm->MD, sizeof(mm->MD));
         printf("\n");
         break;
+
+    default:
+        printf("DF:%u", mm->msgtype);
+        break;
     }
 
     printf(" %s", df_to_string(mm->msgtype));

mode_s.h

@@ -2,7 +2,7 @@
 //
 // mode_s.h: Mode S message decoding (prototypes)
 //
-// Copyright (c) 2017 FlightAware, LLC
+// Copyright (c) 2017-2021 FlightAware, LLC
 // Copyright (c) 2017 Oliver Jowett <oliver@mutability.co.uk>
 //
 // This file is free software: you may copy, redistribute and/or modify it
@@ -26,9 +26,62 @@
 //
 // Functions exported from mode_s.c
 //
+
+// Possible return values of scoreModesMessage,
+// ordered from worst to best
+typedef enum {
+    SR_NOT_SET = 0,               // message has not been scored yet
+
+    SR_ALL_ZEROS,                 // a message that's all zeros
+    SR_UNKNOWN_DF,                // message with unrecognized DF
+    SR_UNCORRECTABLE,             // message with uncorrectable errors
+
+    SR_UNKNOWN_THRESHOLD,         // cutoff for message that might be valid, but don't match an existing aircraft
+
+    SR_UNRELIABLE_UNKNOWN,        // Address/Parity,                 unknown aircraft
+
+    SR_DF11_IID_1ERROR_UNKNOWN,   // DF11, non-zero IID, 1 error,    unknown aircraft
+    SR_DF11_ACQ_1ERROR_UNKNOWN,   // DF11, zero IID,     1 error,    unknown aircraft
+    SR_DF11_IID_UNKNOWN,          // DF11, non-zero IID, no errors,  unknown aircraft
+
+    SR_DF18_2ERROR_UNKNOWN,       // DF17,               2 errors,   unknown aircraft
+    SR_DF17_2ERROR_UNKNOWN,       // DF17,               2 errors,   unknown aircraft
+
+    SR_ACCEPT_THRESHOLD,          // cutoff for accepting messages
+
+    // Address/Parity is unreliable, prefer anything else but this
+    SR_UNRELIABLE_KNOWN,          // Address/Parity,                 known aircraft
+
+    // 2-bit error correction is quite unreliable, put it low down the ranking even for known aircraft
+    SR_DF18_2ERROR_KNOWN,         // DF18,               2 errors,   known aircraft
+    SR_DF17_2ERROR_KNOWN,         // DF17,               2 errors,   known aircraft
+
+    // 1-bit error when we haven't previously seen anything from this address, low priority
+    SR_DF18_1ERROR_UNKNOWN,       // DF18,               1 error,    unknown aircraft
+    SR_DF17_1ERROR_UNKNOWN,       // DF17,               1 error,    unknown aircraft
+
+    // We need to accept at least one non-ES message type from unknown aircraft
+    // or else we'd never accept message from Mode-S only aircraft
+    SR_DF11_ACQ_UNKNOWN,          // DF11, zero IID,     no errors,  unknown aircraft
+
+    SR_DF11_IID_1ERROR_KNOWN,     // DF11, non-zero IID, 1 error,    known aircraft
+    SR_DF11_ACQ_1ERROR_KNOWN,     // DF11, zero IID,     1 error,    known aircraft
+    SR_DF11_IID_KNOWN,            // DF11, non-zero IID, no errors,  known aircraft
+
+    SR_DF18_1ERROR_KNOWN,         // DF18,               1 error,    known aircraft
+    SR_DF17_1ERROR_KNOWN,         // DF17,               1 error,    known aircraft
+
+    SR_DF11_ACQ_KNOWN,            // DF11, zero IID,     no errors,  known aircraft
+
+    SR_DF18_UNKNOWN,              // DF18,               no errors,  unknown aircraft
+    SR_DF17_UNKNOWN,              // DF17,               no errors,  unknown aircraft
+    SR_DF18_KNOWN,                // DF18,               no errors,  known aircraft
+    SR_DF17_KNOWN,                // DF17,               no errors,  known aircraft
+} score_rank;
+
 int modesMessageLenByType(int type);
-int scoreModesMessage(unsigned char *msg, int validbits);
+score_rank scoreModesMessage(const unsigned char *msg);
-int decodeModesMessage (struct modesMessage *mm, unsigned char *msg);
+int decodeModesMessage (struct modesMessage *mm, const unsigned char *msg);
 void displayModesMessage(struct modesMessage *mm);
 void useModesMessage    (struct modesMessage *mm);
 
@@ -38,7 +91,7 @@ void useModesMessage    (struct modesMessage *mm);
 // with how the specs number them.
 
 // Extract one bit from a message.
-static inline  __attribute__((always_inline)) unsigned getbit(unsigned char *data, unsigned bitnum)
+static inline  __attribute__((always_inline)) unsigned getbit(const unsigned char *data, unsigned bitnum)
 {
     unsigned bi = bitnum - 1;
     unsigned by = bi >> 3;
@@ -48,7 +101,7 @@ static inline  __attribute__((always_inline)) unsigned getbit(unsigned char *dat
 }
 
 // Extract some bits (firstbit .. lastbit inclusive) from a message.
-static inline  __attribute__((always_inline)) unsigned getbits(unsigned char *data, unsigned firstbit, unsigned lastbit)
+static inline  __attribute__((always_inline)) unsigned getbits(const unsigned char *data, unsigned firstbit, unsigned lastbit)
 {
     unsigned fbi = firstbit - 1;
     unsigned lbi = lastbit - 1;

tools/df-correction-arrays.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# For each error-correction level (0, 1, or 2-bit errors)
+# generate the set of possible DF values that could possibly be
+# corrected to DF11/17/18/19 messages, expressed as a bitset.
+
+def popcount(x):
+    return bin(x).count('1')
+
+# is the Hamming distance between 'df' and a valid long-message DF no more than `max_errors`?
+def correctable_long(df, max_errors):
+    if popcount(df ^ 17) <= max_errors: return True
+    if popcount(df ^ 18) <= max_errors: return True
+    #if popcount(df ^ 19) <= max_errors: return True
+    return False
+
+# is the Hamming distance between 'df' and a valid short-message DF no more than `max_errors`?
+def correctable_short(df, max_errors):
+    if popcount(df ^ 11) <= max_errors: return True
+    return False
+
+# Generate a bitset value where bit N is set if predicate(N) is True
+def bitset(predicate):
+    result = 0
+    for i in range(32):
+        if predicate(i):
+            result |= 1 << i
+    return result
+
+shorts = [
+    bitset(lambda x: correctable_short(x,0)),
+    bitset(lambda x: correctable_short(x,1)),
+    bitset(lambda x: correctable_short(x,1))  # deliberately not 2
+]
+
+longs = [
+    bitset(lambda x: correctable_long(x,0)),
+    bitset(lambda x: correctable_long(x,1)),
+    bitset(lambda x: correctable_long(x,2))
+]
+
+print('static const uint32_t df_correctable_short[MODES_MAX_BITERRORS + 1] = {')
+print('    ' + ', '.join(f'0x{i:08x}' for i in shorts))
+print('};')
+
+print('static const uint32_t df_correctable_long[MODES_MAX_BITERRORS + 1] = {')
+print('    ' + ', '.join(f'0x{i:08x}' for i in longs))
+print('};')