Determine Page Encoding from Meta Tag

[kevingy]

As discussed in Issue #136, SimpleBrowser currently only sets the page encoding if the character set is specified in the Content-Type HTTP header. If the Content-Type HTTP header specifies the character set, this is the correct behavior and this value should be used. If it is not set in the Content-Type HTTP header, the character set may be specified in HTML:

HTML 4.01: <meta http-equiv="content-type" content="text/html; charset=UTF-8">
HTML5: <meta charset="UTF-8">
XHTML: <?xml version="1.0" encoding="utf-8"?>

See the HTML5 Specification on this topic:
http://www.w3.org/TR/html5/document-metadata.html#character-encoding-declaration

Notes:

1. Implementing this change will require pre-processing (parsing) the first 1K of the received HTML. To improve performance, this code should NOT be executed if the Content-Type specifies the content type.
2. Pre-parsing HTML to determine the character set should not change how parsing the main HTML content works. Currently, HTML is not parsed until the first call to Find(). This is desirable behavior.

[edesantis]

Hi team,

I'm using this library and i think is the best. My solution for the page encoding is add a property "ResponseEncoding" for test the resulting html like chrome (settings>tools>select encoding). In the response stream add the correct encoding or force it by the ResponseEncoding property.

public class Browser
{
public Encoding ResponseEncoding { get; set; }

    internal bool DoRequest(Uri uri, string method, NameValueCollection userVariables, string postData, string contentType, string encodingType, int timeoutMilliseconds)
    {
        string html;
        string referer = null;

        if (uri.IsFile)
        {
            StreamReader reader = new StreamReader(uri.AbsolutePath);
            html = reader.ReadToEnd();
            reader.Close();

            _lastRequestLog = new HttpRequestLog
            {
                Method = "GET",
                Url = uri,
                Text = html
            };
        }
        else
        {
            bool handle3xxRedirect = false;
            int maxRedirects = 5; // Per RFC2068, Section 10.3
            string postBody = string.Empty;
            do
            {
                Debug.WriteLine(uri.ToString());
                if (maxRedirects-- < 0)
                {
                    Log("Too many 3xx redirects", LogMessageType.Error);
                    return false;
                }

                handle3xxRedirect = false;
                IHttpWebRequest req = null;

                try
                {
                    req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
                }
                catch (NotSupportedException)
                {
                    // Happens when the URL cannot be parsed (example: 'javascript:')
                    return false;
                }

                foreach (var header in _extraHeaders)
                {
                    if (header.StartsWith("host:", StringComparison.OrdinalIgnoreCase))
                        req.Host = header.Split(':')[1];
                    else
                        req.Headers.Add(header);
                }

                if (!string.IsNullOrEmpty(encodingType))
                {
                    req.Headers.Add(HttpRequestHeader.ContentEncoding, encodingType);
                }

                // Remove all expired basic authentication tokens
                List<BasicAuthenticationToken> expired =
                    _basicAuthenticationTokens.Values.Where(t => DateTime.Now > t.Expiration).ToList();
                foreach (var expiredToken in expired)
                {
                    _basicAuthenticationTokens.Remove(expiredToken.Domain);
                }

                // If an authentication token exists for the domain, add the authorization header.
                foreach (var token in _basicAuthenticationTokens.Values)
                {
                    if (req.Host.Contains(token.Domain))
                    {
                        // Extend the expiration.
                        token.UpdateExpiration();

                        // Add the authentication header.
                        req.Headers.Add(string.Format(
                            "Authorization: Basic {0}",
                            token.Token));
                    }
                }

                if (_includeFormValues != null)
                {
                    if (userVariables == null)
                        userVariables = _includeFormValues;
                    else
                        userVariables.Add(_includeFormValues);
                }

                if (userVariables != null)
                {
                    if (method == "POST")
                    {
                        postBody = StringUtil.MakeQueryString(userVariables);
                        byte[] data = Encoding.GetEncoding(28591).GetBytes(postBody);
                        req.ContentLength = data.Length;
                        using (Stream stream = req.GetRequestStream())
                        {
                            stream.Write(data, 0, data.Length);
                        }
                    }
                    else
                    {
                        uri = new Uri(
                            uri.Scheme + "://" + uri.Host + ":" + uri.Port + uri.AbsolutePath
                            + ((userVariables.Count > 0) ? "?" + StringUtil.MakeQueryString(userVariables) : "")
                            );
                        req = PrepareRequestObject(uri, method, contentType, timeoutMilliseconds);
                    }
                }
                else if (postData != null)
                {
                    if (method == "GET")
                        throw new InvalidOperationException("Cannot call DoRequest with method GET and non-null postData");
                    postBody = postData;
                    // 28591 corresponds to ISO-8859-1, the default encoding of an HTTP POST.
                    // http://msdn.microsoft.com/en-us/library/system.text.encodinginfo.getencoding%28v=vs.110%29.aspx
                    // In the event that this value ever changes, including no longer being hard coded,
                    // update the encoding being sent with a form submission with a hidden input named _charset_
                    // in InputElement.cs.
                    byte[] data = Encoding.GetEncoding(28591).GetBytes(postData);
                    req.ContentLength = data.Length;
                    using (Stream stream = req.GetRequestStream())
                    {
                        stream.Write(data, 0, data.Length);
                    }
                }

                referer = req.Referer;

                if (contentType != null)
                    req.ContentType = contentType;

                _lastRequestLog = new HttpRequestLog
                {
                    Method = method,
                    PostData = userVariables,
                    PostBody = postBody,
                    RequestHeaders = req.Headers,
                    Url = uri
                };
                try
                {
                    using (IHttpWebResponse response = req.GetResponse())
                    {
                        Encoding responseEncoding = ResponseEncoding ?? Encoding.UTF8; //default
                        if (ResponseEncoding == null &&
                            ((response.Headers.AllKeys.Contains("Content-Type", StringComparer.OrdinalIgnoreCase) &&
                             response.Headers["Content-Type"].IndexOf("charset", 0, StringComparison.OrdinalIgnoreCase) > -1) ||
                             !string.IsNullOrWhiteSpace(response.CharacterSet)))
                        {
                            try
                            {
                                responseEncoding = Encoding.GetEncoding(response.CharacterSet);
                            }
                            catch (ArgumentException)
                            {
                                responseEncoding = Encoding.UTF8; // try using utf8
                            }
                        }

                        //ensure the stream is disposed
                        using (Stream rs = response.GetResponseStream())
                        {
                            using (StreamReader reader = new StreamReader(rs, responseEncoding))
                            {
                                html = reader.ReadToEnd();
                            }
                        }

                        _doc = null;
                        _includeFormValues = null;

                        _lastRequestLog.Text = html;
                        _lastRequestLog.ResponseHeaders = response.Headers;
                        _lastRequestLog.ResponseCode = (int)response.StatusCode;

                        if (method == "GET" && uri.Query.Length > 0 && uri.Query != "?")
                        {
                            _lastRequestLog.QueryStringData = HttpUtility.ParseQueryString(uri.Query);
                        }

                        if (AutoRedirect == true &&
                            (((int)response.StatusCode == 300 || // Not entirely supported. If provided, the server's preference from the Location header is honored.
                            (int)response.StatusCode == 301 ||
                            (int)response.StatusCode == 302 ||
                            (int)response.StatusCode == 303 ||
                            // 304 - Unsupported, conditional Get requests are not supported (mostly because SimpleBrowser does not cache content)
                            // 305 - Unsupported, possible security threat
                            // 306 - No longer used, per RFC2616, Section 10.3.7
                            (int)response.StatusCode == 307 ||
                            (int)response.StatusCode == 308) &&
                            response.Headers.AllKeys.Contains("Location")))
                        {
                            uri = new Uri(uri, response.Headers["Location"]);
                            handle3xxRedirect = true;
                            Debug.WriteLine("Redirecting to: " + uri);
                            method = "GET";
                            postData = null;
                            userVariables = null;
                        }

                        if (response.Headers.AllKeys.Contains("Set-Cookie"))
                        {
                            var cookies = SetCookieHeaderParser.GetAllCookiesFromHeader(uri.Host, response.Headers["Set-Cookie"]);
                            Cookies.Add(cookies);
                        }
                    }
                }
                catch (WebException ex)
                {
                    if (ex.Response != null)
                    {
                        _lastRequestLog.ResponseHeaders = ex.Response.Headers;
                        //ensure the stream is disposed
                        using (Stream rs = ex.Response.GetResponseStream())
                        {
                            using (StreamReader reader = new StreamReader(rs))
                            {
                                html = reader.ReadToEnd();
                            }
                        }
                        _lastRequestLog.Text = html;
                        _lastRequestLog.ResponseCode = (int)((HttpWebResponse)ex.Response).StatusCode;
                    }

                    LastWebException = ex;

                    switch (ex.Status)
                    {
                        case WebExceptionStatus.Timeout:
                            Log("A timeout occurred while trying to load the web page", LogMessageType.Error);
                            break;

                        case WebExceptionStatus.ReceiveFailure:
                            Log("The response was cut short prematurely", LogMessageType.Error);
                            break;

                        default:
                            Log("An exception was thrown while trying to load the page: " + ex.Message, LogMessageType.Error);
                            break;
                    }
                    return false;
                }
                finally
                {
                    LogRequestData();
                }
            }
            while (handle3xxRedirect);
        }

        this._navigationAttributes = null;
        this.RemoveChildBrowsers(); //Any frames contained in the previous state should be removed. They will be recreated if we ever navigate back
        this.AddNavigationState(
            new NavigationState()
            {
                Html = html,
                Url = uri,
                ContentType = contentType,
                Referer = string.IsNullOrEmpty(referer) ? null : new Uri(Uri.UnescapeDataString(referer))
            });

        return true;
    }
}

}

[kevingy]

This is a good approach, although it doesn't really address the original issue. Can you submit a pull request with these changes? Thanks!

[kevingy]

@edesantis Your change has been merged into the code base. Thanks!

[edesantis]

im sorry for not make a fork, but i have a lot of work this weeks. Thanks to all the team for this excelent library.