Further refinements to checking of HTML5 documentsorigin/sideshowbarker/html5-updates

Tested and found to be working as expected.

This causes all validation requests for HTML5 documents to just get
re-posted as-is to https://validator.w3.org/nu/ using an HTTP 307
"Temporary Redirect".＊ —with the exception of any requests that have
`output=soap12` or `output=ucn` parameters.

The reason for making temporary exceptions for those soap12/ucn cases is that
some existing tools such as Dreamweaver apparently have integrated support
for HTML checking that relies on calling https://validator.w3.org/ to do
the validation, and that expects the results in the `output=soap12` format.

Other than those exceptions, this change properly 307-redirects all requests
for checking of HTML5 documents to https://validator.w3.org/nu/ (that
includes file-upload requests as well direct-input requests and URL-input
GET requests, and any `output=json` requests regardless of method).

＊Note: Using a 307 gives the meaning "I'm redirecting this particular
request to https://validator.w3.org/nu/ but that doesn't imply you should
necessarily use https://validator.w3.org/nu/ for any further requests",
which seems like the right meaning for this particular case.

4 files changed, 141 insertions, 50 deletions

diff --git a/httpd/cgi-bin/check b/httpd/cgi-bin/check
index 99d97db..2ec6825 100755
--- a/httpd/cgi-bin/check
+++ b/httpd/cgi-bin/check
@@ -28,7 +28,6 @@ use 5.008;
 use strict;
 use warnings;
 use utf8;
-use MIME::Base64 ();
 
 package W3C::Validator::MarkupValidator;
 
@@ -622,12 +621,6 @@ $File->{WF_Errors}  = [];
 if (($File->{DOCTYPE} eq "HTML5") or ($File->{DOCTYPE} eq "XHTML5")) {
     if ($CFG->{External}->{HTML5}) {
         $File = &html5_validate($File);
-        &add_warning(
-            'W00',
-            {   W00_experimental_name => "HTML5 Conformance Checker",
-                W00_experimental_URI  => "feedback.html"
-            }
-        );
     }
     else {
         $File->{'Error Flagged'} = TRUE;
@@ -644,12 +637,6 @@ elsif (($File->{DOCTYPE} eq '') and
     # namespaces found, to a different engine. WARNING this is experimental.
     if ($CFG->{External}->{CompoundXML}) {
         $File = &compoundxml_validate($File);
-        &add_warning(
-            'W00',
-            {   W00_experimental_name => "validator.nu Conformance Checker",
-                W00_experimental_URI  => "feedback.html"
-            }
-        );
     }
 }
 else {
@@ -1098,21 +1085,8 @@ sub html5_validate (\$)
         $req->header('Accept-Encoding', 'identity');
     }
 
-    my $source_option = $File->{Opt}->{'Show Source'} ? "&showsource=yes" : "";
-    my $outline_option = $File->{Opt}->{Outline} ? "&showoutline=yes" : "";
-    my $output_option = $File->{Opt}->{Output} eq 'json' ? "&out=json" : "";
-    my $uri = uri_escape($File->{'URI'});
-    if ($File->{'Direct Input'}) {
-        # if $req isn't actually encoded, this decode() call does nothing
-        $req->decode("gzip");
-        $uri = "data:text/html;charset=utf-8;base64," .
-            uri_escape(MIME::Base64::encode_base64($req->content));
-    }
-    if (!$File->{'Is Upload'}) {
-        print redirect
-            'https://validator.w3.org/nu/?doc=' . $uri .
-            $source_option . $outline_option . $output_option;
-    }
+    redirect_html5_requests();
+
     my $res = $ua->request($req);
     if (!$res->is_success()) {
         $File->{'Error Flagged'} = TRUE;
@@ -1457,6 +1431,10 @@ sub prep_template ($$)
     my $T    = shift;
 
     #
+    # URL for Nu Html Checker
+    $T->param(htmlchecker_url => $CFG->{External}->{HTML5});
+
+    #
     # XML mode...
     $T->param(is_xml => &is_xml($File));
 
@@ -1591,6 +1569,9 @@ sub fin_template ($$)
     }
     my ($num_errors, $num_warnings, $num_info, $reported_errors) =
         &report_errors($File);
+    if ($File->{Version} eq "HTML5") {
+        $num_warnings++;
+    }
     if ($num_errors + $num_warnings > 0) {
         $T->param(has_errors => 1);
     }
@@ -3378,6 +3359,25 @@ sub self_url_file
     return $thispage;
 }
 
+sub redirect_html5_requests
+{
+    return if ($File->{Opt}->{Output} eq 'soap12');
+    return if ($File->{Opt}->{Output} eq 'ucn');
+    return if not($CFG->{External}->{HTML5});
+    if ($File->{'Direct Input'} || $File->{'Is Upload'}) {
+        my $hash = $File->{'Direct Input'} ? "#textarea" : "#file";
+        print redirect(
+            -uri => $CFG->{External}->{HTML5} . $hash,
+            -status => '307 Temporary Redirect');
+    }
+    my $source_option = $File->{Opt}->{'Show Source'} ? "&showsource=yes" : "";
+    my $outline_option = $File->{Opt}->{Outline} ? "&showoutline=yes" : "";
+    my $output_option = $File->{Opt}->{Output} eq 'json' ? "&out=json" : "";
+    print redirect
+        $CFG->{External}->{HTML5} . '?doc=' . uri_escape($File->{'URI'}) .
+        $source_option . $outline_option . $output_option;
+}
+
 #####
 
 package W3C::Validator::EventHandler;
diff --git a/share/templates/en_US/error_messages.cfg b/share/templates/en_US/error_messages.cfg
index 815f2ff..1f58987 100644
--- a/share/templates/en_US/error_messages.cfg
+++ b/share/templates/en_US/error_messages.cfg
@@ -2038,3 +2038,33 @@ verbose <<.EOF.
     </div>
   .EOF.
 </msg>
+<msg obsolete-interface>
+  original = "This interface to HTML5 document checking is obsolete. Use an interface to the W3C Nu Html Checker instead."
+  verbose <<.EOF.
+    <div class="ve obsolete-interface">
+      <p>
+        The method you used to check this document relies on an obsolete interface
+        that will become permanently unavailable in the near future.
+        The currently-supported method is to instead use the W3C Nu Html Checker at
+        <a href="https://validator.w3.org/nu/">https://validator.w3.org/nu/</a>.
+        See the documentation on the W3C Nu Html Checker's
+        <a href="https://github.com/validator/validator/wiki/Service:-Input:-GET">GET interface</a>,
+        <a href="https://github.com/validator/validator/wiki/Service:-Input:-POST-body">POST interface</a>,
+        and
+        <a href="https://github.com/validator/validator/wiki/Service:-Common-parameters">the parameters</a>
+        that both interfaces provide, such as the
+        <a href="https://github.com/validator/validator/wiki/Output%3A-JSON">out=json</a>,
+        <a href="https://github.com/validator/validator/wiki/Output:-XML">out=xml</a>,
+        and
+        <a href="https://github.com/validator/validator/wiki/Output:-GNU">out=gnu</a>
+        parameters/output-formats.
+      </p>
+      <p>
+        For comments or questions about this message, send mail to
+        <a href="mailto:www-validator@w3.org">www-validator@w3.org</a>
+        or to
+        <a href="https://twitter.com/w3c">@w3c</a> on twitter.
+      </p>
+    </div>
+  .EOF.
+</msg>
diff --git a/share/templates/en_US/soap_output.tmpl b/share/templates/en_US/soap_output.tmpl
index 7b3dec9..a87ff0a 100644
--- a/share/templates/en_US/soap_output.tmpl
+++ b/share/templates/en_US/soap_output.tmpl
@@ -16,10 +16,60 @@ X-W3C-Validator-Warnings: <TMPL_VAR NAME="valid_warnings_num"></TMPL_IF>
     <m:doctype><TMPL_VAR NAME="file_doctype" ESCAPE="HTML"></m:doctype>
     <m:charset><TMPL_VAR NAME="file_charset" ESCAPE="HTML"></m:charset>
     <m:validity><TMPL_IF NAME="VALID">true<TMPL_ELSE>false</TMPL_IF></m:validity>
+    <m:warnings>
+        <m:warningcount><TMPL_VAR NAME="valid_warnings_num"></m:warningcount>
+        <m:warninglist>
+           <TMPL_IF NAME="is_html5">  <m:warning>
+                <m:message>This interface to HTML5 document checking is obsolete. Use an interface to <TMPL_VAR NAME="htmlchecker_url"> instead.</m:message>
+                <m:messageid>obsolete-interface</m:messageid>
+                <m:explanation>  <![CDATA[<TMPL_VAR NAME="expl">
+                    <div class="ve obsolete-interface">
+                      <p>
+                        The method you used to check this document relies on an obsolete interface
+                        that will become permanently unavailable in the near future.
+                        The currently-supported method is to instead use the W3C Nu Html Checker at
+                        <a href="<TMPL_VAR NAME='htmlchecker_url'>"><TMPL_VAR NAME="htmlchecker_url"></a>.
+                        See the documentation on the W3C Nu Html Checker's
+                        <a href="https://github.com/validator/validator/wiki/Service:-Input:-GET">GET interface</a>,
+                        <a href="https://github.com/validator/validator/wiki/Service:-Input:-POST-body">POST interface</a>,
+                        and
+                        <a href="https://github.com/validator/validator/wiki/Service:-Common-parameters">the parameters</a>
+                        that both interfaces provide, such as the
+                        <a href="https://github.com/validator/validator/wiki/Output%3A-JSON">out=json</a>,
+                        <a href="https://github.com/validator/validator/wiki/Output:-XML">out=xml</a>,
+                        and
+                        <a href="https://github.com/validator/validator/wiki/Output:-GNU">out=gnu</a>
+                        parameters/output-formats.
+                      </p>
+                      <p>
+                        For comments or questions about this message, send mail to
+                        <a href="mailto:www-validator@w3.org">www-validator@w3.org</a>
+                        or to
+                        <a href="https://twitter.com/w3c">@w3c</a> on twitter.
+                      </p>
+                    </div>
+                  ]]>
+                </m:explanation>
+                <m:source></m:source>
+            </m:warning></TMPL_IF><TMPL_IF NAME="have_warnings"><TMPL_INCLUDE NAME="soap_warnings.tmpl"></TMPL_IF>
+        <TMPL_LOOP NAME="file_errors"><TMPL_IF NAME="err_type_warn">
+            <m:warning>
+                <m:line><TMPL_VAR NAME="line"></m:line>
+                <m:col><TMPL_VAR NAME="char" ESCAPE="HTML"></m:col>
+                <m:message><TMPL_VAR NAME="msg" ESCAPE="HTML"></m:message>
+                  <m:messageid><TMPL_VAR NAME="num" ESCAPE="HTML"></m:messageid>
+                  <m:explanation>  <![CDATA[
+                    <TMPL_VAR NAME="expl">
+                    ]]>
+                  </m:explanation>
+                  <m:source><![CDATA[<TMPL_VAR NAME="src">]]></m:source>
+            </m:warning>
+        </TMPL_IF></TMPL_LOOP></m:warninglist>
+    </m:warnings>
     <m:errors>
         <m:errorcount><TMPL_VAR NAME="valid_errors_num"></m:errorcount>
         <m:errorlist>
-          <TMPL_LOOP NAME="file_errors"><TMPL_IF NAME="err_type_err">
+<TMPL_LOOP NAME="file_errors"><TMPL_IF NAME="err_type_err">
             <m:error>
                 <m:line><TMPL_VAR NAME="line"></m:line>
                 <m:col><TMPL_VAR NAME="char" ESCAPE="HTML"></m:col>
@@ -34,25 +84,6 @@ X-W3C-Validator-Warnings: <TMPL_VAR NAME="valid_warnings_num"></TMPL_IF>
            </TMPL_IF></TMPL_LOOP>
         </m:errorlist>
     </m:errors>
-    <m:warnings>
-        <m:warningcount><TMPL_VAR NAME="valid_warnings_num"></m:warningcount>
-        <m:warninglist>
-        <TMPL_IF NAME="have_warnings"><TMPL_INCLUDE NAME="soap_warnings.tmpl"></TMPL_IF>
-        <TMPL_LOOP NAME="file_errors"><TMPL_IF NAME="err_type_warn">
-            <m:warning>
-                <m:line><TMPL_VAR NAME="line"></m:line>
-                <m:col><TMPL_VAR NAME="char" ESCAPE="HTML"></m:col>
-                <m:message><TMPL_VAR NAME="msg" ESCAPE="HTML"></m:message>
-                  <m:messageid><TMPL_VAR NAME="num" ESCAPE="HTML"></m:messageid>
-                  <m:explanation>  <![CDATA[
-                    <TMPL_VAR NAME="expl">
-                    ]]>
-                  </m:explanation>
-                  <m:source><![CDATA[<TMPL_VAR NAME="src">]]></m:source>
-            </m:warning>
-          </TMPL_IF></TMPL_LOOP>
-        </m:warninglist>
-    </m:warnings>
 </m:markupvalidationresponse>
 </env:Body>
 </env:Envelope>
diff --git a/share/templates/en_US/ucn_output.tmpl b/share/templates/en_US/ucn_output.tmpl
index 9e9265a..b1f2a52 100644
--- a/share/templates/en_US/ucn_output.tmpl
+++ b/share/templates/en_US/ucn_output.tmpl
@@ -8,7 +8,7 @@ X-W3C-Validator-Warnings: <TMPL_VAR NAME="valid_warnings_num"></TMPL_IF>
 <?xml version="1.0" encoding="UTF-8"?>
 <observationresponse xmlns="http://www.w3.org/2009/10/unicorn/observationresponse" ref="<TMPL_VAR NAME="file_uri" ESCAPE="HTML">" xml:lang="en">
 
-<TMPL_IF NAME="VALID">
+<TMPL_IF NAME="is_html5"><TMPL_ELSE><TMPL_IF NAME="VALID">
 <status value="passed"/>
 
 <message type="info" id="valid">
@@ -102,7 +102,37 @@ X-W3C-Validator-Warnings: <TMPL_VAR NAME="valid_warnings_num"></TMPL_IF>
 		</TMPL_UNLESS>
 	</description>
 </message>
-</TMPL_IF>
+</TMPL_IF></TMPL_IF>
+
+<TMPL_IF NAME="is_html5"><message type="warning" id="obsolete-interface">
+	<context/>
+	<title>This interface to HTML5 document checking is obsolete. Use an interface to <TMPL_VAR NAME="htmlchecker_url"> instead.</title>
+	<description>
+		<p>
+			The method you used to check this document relies on an obsolete interface
+			that will become permanently unavailable in the near future.
+			The currently-supported method is to instead use the W3C Nu Html Checker at
+			<a href="<TMPL_VAR NAME='htmlchecker_url'>"><TMPL_VAR NAME="htmlchecker_url"></a>.
+			See the documentation on the W3C Nu Html Checker's
+			<a href="https://github.com/validator/validator/wiki/Service:-Input:-GET">GET interface</a>,
+			<a href="https://github.com/validator/validator/wiki/Service:-Input:-POST-body">POST interface</a>,
+			and
+			<a href="https://github.com/validator/validator/wiki/Service:-Common-parameters">the parameters</a>
+			that both interfaces provide, such as the
+			<a href="https://github.com/validator/validator/wiki/Output%3A-JSON">out=json</a>,
+			<a href="https://github.com/validator/validator/wiki/Output:-XML">out=xml</a>,
+			and
+			<a href="https://github.com/validator/validator/wiki/Output:-GNU">out=gnu</a>
+			parameters/output-formats.
+		</p>
+		<p>
+			For comments or questions about this message, send mail to
+			<a href="mailto:www-validator@w3.org">www-validator@w3.org</a>
+			or to
+			<a href="https://twitter.com/w3c">@w3c</a> on twitter.
+		</p>
+	</description>
+</message></TMPL_IF>
 
 <TMPL_IF NAME="have_warnings">
 	<TMPL_INCLUDE NAME="ucn_warnings.tmpl">