Applied topology! Instead of using the standard even-odd raycasting trick for winding number (which I didn’t remember), I constructed an explicit inward-pointing normal at each edge of the large polygon. This way you can test directly whether edges that lie on the boundary of your candidate rectangle have the right orientation. The code isn’t pretty (edge-exterior-to-rect? is particularly awful), and I’m sure it could be made faster, but this worked almost the first time.

(ql:quickload :str)

(defun parse-line (line)
  (map 'vector #'parse-integer (str:split "," line)))

(defun read-inputs (filename)
  (let ((lines (uiop:read-file-lines filename)))
    (make-array (list (length lines) 2)
                :initial-contents (mapcar #'parse-line lines))))

(defun area (corners v w)
  "Yield the area of the rectangle between two diagonally opposite corners identified by index."
  (* (1+ (abs (- (aref corners v 0) (aref corners w 0))))
     (1+ (abs (- (aref corners v 1) (aref corners w 1))))))

(defun main-1 (filename)
  (let* ((red-tiles (read-inputs filename))
         (ntiles (car (array-dimensions red-tiles))))
    (loop for v from 0 to (1- ntiles)
          maximize (loop for w from 0 to (1- v)
                         maximize (area red-tiles v w)))))

;;; A rectangle is enclosed by the region if and only if both of the following are true:
;;; 1. No edge of the region passes through the interior of the rectangle;
;;; 2. When an edge of the region runs along an edge of the rectangle, the interior of
;;;    the region and the interior of the rectangle lie on the same side.
;;; So our first problem is to find the interior side of each edge of the region.

(defun edges-with-interior (corners)
  ;; Anchor: a vertical edge with minimum x coordinate has the interior on the +x side
  (let* ((n (car (array-dimensions corners)))
         (min-x (loop for i from 0 to (1- n) minimize (aref corners i 0)))
         (arg-min-x (loop for i from 0 to (1- n)
                          when (= (aref corners i 0) min-x)
                            return i))
         ;; Mapping from edge direction unit vector to inward-pointing normal to edge
         ;; Initial inward normal is known to be #(1 0); sense of the rotation from edge
         ;; direction depends on direction of initial edge
         (edge-interior-dir
           (if (> (aref corners (1+ arg-min-x) 1) (aref corners arg-min-x 1)) ; +y
               '((#(0 1) . #(1 0)) (#(1 0) . #(0 -1)) (#(0 -1) . #(-1 0)) (#(-1 0) . #(0 1)))
               '((#(0 -1) . #(1 0)) (#(1 0) . #(0 1)) (#(0 1) . #(-1 0)) (#(-1 0) . #(0 -1)))))
         (interior-labeled-edges
           (loop for i from arg-min-x to (+ arg-min-x n -1)
                 collect (let* ((j (mod i n))
                                (jj (mod (1+ i) n))
                                (unit-edge
                                  (vector (signum (- (aref corners jj 0) (aref corners j 0)))
                                          (signum (- (aref corners jj 1) (aref corners j 1)))))
                                (unit-interior-normal
                                  (cdr (assoc unit-edge edge-interior-dir :test #'equalp))))
                           (vector (aref corners j 0) (aref corners j 1)
                                   (aref corners jj 0) (aref corners jj 1)
                                   (aref unit-interior-normal 0) (aref unit-interior-normal 1))))))
    (make-array (list n 6) :initial-contents interior-labeled-edges)))

(defun edge-exterior-to-rect? (corners edges-with-interior i v w)
  (let ((rxmin (min (aref corners v 0) (aref corners w 0)))
        (rxmax (max (aref corners v 0) (aref corners w 0)))
        (rymin (min (aref corners v 1) (aref corners w 1)))
        (rymax (max (aref corners v 1) (aref corners w 1)))
        (exmin (min (aref edges-with-interior i 0) (aref edges-with-interior i 2)))
        (exmax (max (aref edges-with-interior i 0) (aref edges-with-interior i 2)))
        (eymin (min (aref edges-with-interior i 1) (aref edges-with-interior i 3)))
        (eymax (max (aref edges-with-interior i 1) (aref edges-with-interior i 3)))
        (ixdir (aref edges-with-interior i 4))
        (iydir (aref edges-with-interior i 5)))
    (or (and (< exmin rxmin) (<= exmax rxmin)) ; edge is left of rectangle
        (and (>= exmin rxmax) (> exmax rxmax)) ; edge is right of rectangle
        (and (< eymin rymin) (<= eymax rymin)) ; edge is above rectangle
        (and (>= eymin rymax) (> eymax rymax)) ; edge is below rectangle
        (and (= exmin exmax rxmin) (= ixdir 1) (= iydir 0)) ; edge lies on left side pointing in
        (and (= exmin exmax rxmax) (= ixdir -1) (= iydir 0)) ; edge lies on right side pointing in
        (and (= eymin eymax rymin) (= ixdir 0) (= iydir 1)) ; edge lies on top side pointing in
        (and (= eymin eymax rymax) (= ixdir 0) (= iydir -1))))) ; edge lies on bottom side pointing in

(defun enclosed? (corners edges-with-interior v w)
  (loop for i from 0 to (1- (car (array-dimensions edges-with-interior)))
        unless (edge-exterior-to-rect? corners edges-with-interior i v w)
          return nil
        finally (return t)))

(defun main-2 (filename)
  (let* ((red-tiles (read-inputs filename))
         (ntiles (car (array-dimensions red-tiles)))
         (edges-with-interior (edges-with-interior red-tiles)))
    (loop for v from 0 to (1- ntiles)
          maximize (loop for w from 0 to (1- v)
                         maximize (if (enclosed? red-tiles edges-with-interior v w)
                                      (area red-tiles v w)
                                      0)))))