Async GL shader compilation

src/Magnum/GL/AbstractShaderProgram.cpp

@@ -585,53 +585,55 @@ void AbstractShaderProgram::transformFeedbackVaryingsImplementationDanglingWorka
 
 bool AbstractShaderProgram::link() { return link({*this}); }
 
-bool AbstractShaderProgram::link(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders) {
-    bool allSuccess = true;
+void AbstractShaderProgram::submitLink() {
+    glLinkProgram(_id);
+}
+
+bool AbstractShaderProgram::checkLink() {
+    GLint success, logLength;
+    glGetProgramiv(_id, GL_LINK_STATUS, &success);
+    glGetProgramiv(_id, GL_INFO_LOG_LENGTH, &logLength);
 
-    /* Invoke (possibly parallel) linking on all shaders */
-    for(AbstractShaderProgram& shader: shaders) glLinkProgram(shader._id);
-
-    /* After linking phase, check status of all shaders */
-    Int i = 1;
-    for(AbstractShaderProgram& shader: shaders) {
-        GLint success, logLength;
-        glGetProgramiv(shader._id, GL_LINK_STATUS, &success);
-        glGetProgramiv(shader._id, GL_INFO_LOG_LENGTH, &logLength);
-
-        /* Error or warning message. The string is returned null-terminated,
-           strip the \0 at the end afterwards. */
-        std::string message(logLength, '\n');
-        if(message.size() > 1)
-            glGetProgramInfoLog(shader._id, message.size(), nullptr, &message[0]);
-        message.resize(Math::max(logLength, 1)-1);
-
-        /* Some drivers are chatty and can't keep shut when there's nothing to
-           be said, handle that as well. */
-        Context::current().state().shaderProgram.cleanLogImplementation(message);
-
-        /* Show error log */
-        if(!success) {
-            Error out{Debug::Flag::NoNewlineAtTheEnd};
-            out << "GL::AbstractShaderProgram::link(): linking";
-            if(shaders.size() != 1) out << "of shader" << i;
-            out << "failed with the following message:" << Debug::newline << message;
-
-        /* Or just warnings, if any */
-        } else if(!message.empty()) {
-            Warning out{Debug::Flag::NoNewlineAtTheEnd};
-            out << "GL::AbstractShaderProgram::link(): linking";
-            if(shaders.size() != 1) out << "of shader" << i;
-            out << "succeeded with the following message:" << Debug::newline << message;
-        }
-
-        /* Success of all depends on each of them */
-        allSuccess = allSuccess && success;
-        ++i;
+    /* Error or warning message. The string is returned null-terminated,
+       strip the \0 at the end afterwards. */
+    std::string message(logLength, '\n');
+    if(message.size() > 1)
+        glGetProgramInfoLog(_id, message.size(), nullptr, &message[0]);
+    message.resize(Math::max(logLength, 1)-1);
+
+    /* Some drivers are chatty and can't keep shut when there's nothing to
+       be said, handle that as well. */
+    Context::current().state().shaderProgram.cleanLogImplementation(message);
+
+    /* Show error log */
+    if(!success) {
+        Error out{Debug::Flag::NoNewlineAtTheEnd};
+        out << "GL::AbstractShaderProgram::link(): linking failed with the following message:"
+            << Debug::newline << message;
+
+    /* Or just warnings, if any */
+    } else if(!message.empty()) {
+        Warning out{Debug::Flag::NoNewlineAtTheEnd};
+        out << "GL::AbstractShaderProgram::link(): linking succeeded with the following message:"
+            << Debug::newline << message;
     }
 
+    return success;
+}
+
+bool AbstractShaderProgram::link(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders) {
+    for(AbstractShaderProgram& shader: shaders) shader.submitLink();
+    bool allSuccess = true;
+    for(AbstractShaderProgram& shader: shaders) allSuccess = allSuccess && shader.checkLink();
     return allSuccess;
 }
 
+bool AbstractShaderProgram::isLinkFinished() {
+    GLint success;
+    Context::current().state().shaderProgram.completionStatusImplementation(_id, GL_COMPLETION_STATUS_KHR, &success);
+    return success == GL_TRUE;
+}
+
 void AbstractShaderProgram::cleanLogImplementationNoOp(std::string&) {}
 
 #if defined(CORRADE_TARGET_WINDOWS) && !defined(MAGNUM_TARGET_GLES)
@@ -646,6 +648,10 @@ void AbstractShaderProgram::cleanLogImplementationAngle(std::string& message) {
 }
 #endif
 
+void AbstractShaderProgram::completionStatusImplementationFallback(GLuint, GLenum, GLint* value) {
+    *value = GL_TRUE;
+}
+
 Int AbstractShaderProgram::uniformLocationInternal(const Containers::ArrayView<const char> name) {
     const GLint location = glGetUniformLocation(_id, name);
     if(location == -1)

src/Magnum/GL/AbstractShaderProgram.h

@@ -1258,19 +1258,26 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
         AbstractShaderProgram& dispatchCompute(const Vector3ui& workgroupCount);
         #endif
 
-    protected:
         /**
-         * @brief Link the shader
+         * @brief Non-blocking linking status check
+         * @return @cpp true @ce if linking finished, @cpp false @ce otherwise
+         *
+         * On some drivers this might return false even after
+         * @ref checkLink() reported successful linking.
          *
+         * @see @fn_gl_keyword{GetProgram} with
+         * @def_gl_extension{COMPLETION_STATUS,KHR,parallel_shader_compile}
+         */
+        bool isLinkFinished();
+
+    protected:
+        /** @brief Link the shaders
+         *
+         * Calls @ref submitLink() on all shaders first, then @ref checkLink().
          * Returns @cpp false @ce if linking of any shader failed, @cpp true @ce
-         * if everything succeeded. Linker message (if any) is printed to error
-         * output. All attached shaders must be compiled with
-         * @ref Shader::compile() before linking. The operation is batched in a
+         * if everything succeeded. The operation is batched in a
          * way that allows the driver to link multiple shaders simultaneously
          * (i.e. in multiple threads).
-         * @see @fn_gl_keyword{LinkProgram}, @fn_gl_keyword{GetProgram} with
-         *      @def_gl{LINK_STATUS} and @def_gl{INFO_LOG_LENGTH},
-         *      @fn_gl_keyword{GetProgramInfoLog}
          */
         static bool link(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders);
 
@@ -1446,13 +1453,37 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
         /**
          * @brief Link the shader
          *
-         * Links single shader. If possible, prefer to link multiple shaders
-         * at once using @ref link(std::initializer_list<Containers::Reference<AbstractShaderProgram>>)
+         * Calls @ref submitLink(), then @ref checkLink().
+         * If possible, prefer to link multiple shaders at once using
+         * @ref link(std::initializer_list<Containers::Reference<AbstractShaderProgram>>)
          * for improved performance, see its documentation for more
          * information.
          */
         bool link();
 
+        /**
+         * @brief Submit for linking
+         *
+         * The attached shaders must be compiled with @ref Shader::compile()
+         * or @ref Shader::submitCompile() before linking.
+         *
+         * @see @fn_gl_keyword{LinkProgram}
+         */
+        void submitLink();
+
+        /**
+         * @brief Check link status and await completion
+         *
+         * Returns @cpp false @ce if linking failed, @cpp true @ce on success.
+         * Linker message (if any) is printed to error output. This function
+         * must be called only after @ref submitLink().
+         *
+         * @see @fn_gl_keyword{GetProgram} with
+         *      @def_gl{LINK_STATUS} and @def_gl{INFO_LOG_LENGTH},
+         *      @fn_gl_keyword{GetProgramInfoLog}
+         */
+        bool checkLink();
+
         /**
          * @brief Get uniform location
          * @param name          Uniform name
@@ -1668,6 +1699,8 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
         static MAGNUM_GL_LOCAL void cleanLogImplementationAngle(std::string& message);
         #endif
 
+        static MAGNUM_GL_LOCAL void completionStatusImplementationFallback(GLuint, GLenum, GLint*);
+
         MAGNUM_GL_LOCAL static void use(GLuint id);
         void use();
 

src/Magnum/GL/Implementation/ShaderProgramState.cpp

@@ -78,6 +78,13 @@ ShaderProgramState::ShaderProgramState(Context& context, Containers::StaticArray
         cleanLogImplementation = &AbstractShaderProgram::cleanLogImplementationNoOp;
     }
 
+    if(context.isExtensionSupported<Extensions::KHR::parallel_shader_compile>()) {
+        extensions[Extensions::KHR::parallel_shader_compile::Index] = Extensions::KHR::parallel_shader_compile::string();
+        completionStatusImplementation = glGetProgramiv;
+    } else {
+        completionStatusImplementation = &AbstractShaderProgram::completionStatusImplementationFallback;
+    }
+
     #ifndef MAGNUM_TARGET_WEBGL
     #ifndef MAGNUM_TARGET_GLES2
     #ifndef MAGNUM_TARGET_GLES

src/Magnum/GL/Implementation/ShaderProgramState.h

@@ -47,6 +47,7 @@ struct ShaderProgramState {
     void(AbstractShaderProgram::*transformFeedbackVaryingsImplementation)(Containers::ArrayView<const std::string>, AbstractShaderProgram::TransformFeedbackBufferMode);
     #endif
     void(*cleanLogImplementation)(std::string&);
+    void(*completionStatusImplementation)(GLuint, GLenum, GLint* value);
 
     #ifndef MAGNUM_TARGET_WEBGL
     void(APIENTRY *uniform1fvImplementation)(GLuint, GLint, GLsizei, const GLfloat*);

src/Magnum/GL/Implementation/ShaderState.cpp

@@ -31,12 +31,13 @@
 
 #include "Magnum/GL/Context.h"
 #include "Magnum/GL/Shader.h"
+#include "Magnum/GL/Extensions.h"
 
 namespace Magnum { namespace GL { namespace Implementation {
 
 using namespace Containers::Literals;
 
-ShaderState::ShaderState(Context& context, Containers::StaticArrayView<Implementation::ExtensionCount, const char*>):
+ShaderState::ShaderState(Context& context, Containers::StaticArrayView<Implementation::ExtensionCount, const char*> extensions):
     maxVertexOutputComponents{}, maxFragmentInputComponents{},
     #if !defined(MAGNUM_TARGET_GLES2) && !defined(MAGNUM_TARGET_WEBGL)
     maxTessellationControlInputComponents{}, maxTessellationControlOutputComponents{}, maxTessellationControlTotalOutputComponents{}, maxTessellationEvaluationInputComponents{}, maxTessellationEvaluationOutputComponents{}, maxGeometryInputComponents{}, maxGeometryOutputComponents{}, maxGeometryTotalOutputComponents{}, maxAtomicCounterBuffers{}, maxCombinedAtomicCounterBuffers{}, maxAtomicCounters{}, maxCombinedAtomicCounters{}, maxImageUniforms{}, maxCombinedImageUniforms{}, maxShaderStorageBlocks{}, maxCombinedShaderStorageBlocks{},
@@ -68,9 +69,12 @@ ShaderState::ShaderState(Context& context, Containers::StaticArrayView<Implement
         cleanLogImplementation = &Shader::cleanLogImplementationNoOp;
     }
 
-    /* Needed only if neither of these ifdefs above hits, but I won't bother
-       crafting the preprocessor logic for this. */
-    static_cast<void>(context);
+     if(context.isExtensionSupported<GL::Extensions::KHR::parallel_shader_compile>()) {
+        extensions[Extensions::KHR::parallel_shader_compile::Index] = Extensions::KHR::parallel_shader_compile::string();
+        completionStatusImplementation = glGetShaderiv;
+    } else {
+        completionStatusImplementation = &Shader::completionStatusImplementationFallback;
+    }
 }
 
 }}}

src/Magnum/GL/Implementation/ShaderState.h

@@ -53,6 +53,7 @@ struct ShaderState {
 
     void(Shader::*addSourceImplementation)(std::string);
     void(*cleanLogImplementation)(std::string&);
+    void(*completionStatusImplementation)(GLuint, GLenum, GLint* value);
 
     GLint maxVertexOutputComponents,
         maxFragmentInputComponents;

src/Magnum/GL/Shader.cpp

@@ -749,74 +749,69 @@ Shader& Shader::addFile(const std::string& filename) {
 
 bool Shader::compile() { return compile({*this}); }
 
-bool Shader::compile(std::initializer_list<Containers::Reference<Shader>> shaders) {
-    bool allSuccess = true;
+void Shader::submitCompile() {
+    CORRADE_ASSERT(_sources.size() > 1, "GL::Shader::compile(): no files added", );
 
-    /* Allocate large enough array for source pointers and sizes (to avoid
-       reallocating it for each of them) */
-    std::size_t maxSourceCount = 0;
-    for(Shader& shader: shaders) {
-        CORRADE_ASSERT(shader._sources.size() > 1, "GL::Shader::compile(): no files added", false);
-        maxSourceCount = Math::max(shader._sources.size(), maxSourceCount);
-    }
     /** @todo ArrayTuple/VLAs */
-    Containers::Array<const GLchar*> pointers(maxSourceCount);
-    Containers::Array<GLint> sizes(maxSourceCount);
+    Containers::Array<const GLchar*> pointers(_sources.size());
+    Containers::Array<GLint> sizes(_sources.size());
 
     /* Upload sources of all shaders */
-    for(Shader& shader: shaders) {
-        for(std::size_t i = 0; i != shader._sources.size(); ++i) {
-            pointers[i] = static_cast<const GLchar*>(shader._sources[i].data());
-            sizes[i] = shader._sources[i].size();
-        }
-
-        glShaderSource(shader._id, shader._sources.size(), pointers, sizes);
+    for(std::size_t i = 0; i != _sources.size(); ++i) {
+        pointers[i] = static_cast<const GLchar*>(_sources[i].data());
+        sizes[i] = _sources[i].size();
     }
 
-    /* Invoke (possibly parallel) compilation on all shaders */
-    for(Shader& shader: shaders) glCompileShader(shader._id);
-
-    /* After compilation phase, check status of all shaders */
-    Int i = 1;
-    for(Shader& shader: shaders) {
-        GLint success, logLength;
-        glGetShaderiv(shader._id, GL_COMPILE_STATUS, &success);
-        glGetShaderiv(shader._id, GL_INFO_LOG_LENGTH, &logLength);
-
-        /* Error or warning message. The string is returned null-terminated,
-           strip the \0 at the end afterwards. */
-        std::string message(logLength, '\0');
-        if(message.size() > 1)
-            glGetShaderInfoLog(shader._id, message.size(), nullptr, &message[0]);
-        message.resize(Math::max(logLength, 1)-1);
-
-        /* Some drivers are chatty and can't keep shut when there's nothing to
-           be said, handle that as well. */
-        Context::current().state().shader.cleanLogImplementation(message);
-
-        /* Show error log */
-        if(!success) {
-            Error out{Debug::Flag::NoNewlineAtTheEnd};
-            out << "GL::Shader::compile(): compilation of" << shaderName(shader._type) << "shader";
-            if(shaders.size() != 1) out << i;
-            out << "failed with the following message:" << Debug::newline << message;
-
-        /* Or just warnings, if any */
-        } else if(!message.empty()) {
-            Warning out{Debug::Flag::NoNewlineAtTheEnd};
-            out << "GL::Shader::compile(): compilation of" << shaderName(shader._type) << "shader";
-            if(shaders.size() != 1) out << i;
-            out << "succeeded with the following message:" << Debug::newline << message;
-        }
-
-        /* Success of all depends on each of them */
-        allSuccess = allSuccess && success;
-        ++i;
+    glShaderSource(_id, _sources.size(), pointers, sizes);
+    glCompileShader(_id);
+}
+
+bool Shader::checkCompile() { /* After compilation phase, check status of all shaders */
+    GLint success, logLength;
+    glGetShaderiv(_id, GL_COMPILE_STATUS, &success);
+    glGetShaderiv(_id, GL_INFO_LOG_LENGTH, &logLength);
+
+    /* Error or warning message. The string is returned null-terminated,
+       strip the \0 at the end afterwards. */
+    std::string message(logLength, '\0');
+    if(message.size() > 1)
+        glGetShaderInfoLog(_id, message.size(), nullptr, &message[0]);
+    message.resize(Math::max(logLength, 1)-1);
+
+    /* Some drivers are chatty and can't keep shut when there's nothing to
+       be said, handle that as well. */
+    Context::current().state().shader.cleanLogImplementation(message);
+
+    /* Show error log */
+    if(!success) {
+        Error out{Debug::Flag::NoNewlineAtTheEnd};
+        out << "GL::Shader::compile(): compilation of" << shaderName(_type) << "shader"
+            << "failed with the following message:" << Debug::newline << message;
+
+    /* Or just warnings, if any */
+    } else if(!message.empty()) {
+        Warning out{Debug::Flag::NoNewlineAtTheEnd};
+        out << "GL::Shader::compile(): compilation of" << shaderName(_type) << "shader"
+            << "succeeded with the following message:" << Debug::newline << message;
     }
 
+    return success;
+}
+
+bool Shader::compile(std::initializer_list<Containers::Reference<Shader>> shaders) {
+    /* Invoke (possibly parallel) compilation on all shaders */
+    for(Shader& shader: shaders) shader.submitCompile();
+    bool allSuccess = true;
+    for(Shader& shader: shaders) allSuccess = allSuccess && shader.checkCompile();
     return allSuccess;
 }
 
+bool Shader::isCompileFinished() {
+    GLint success;
+    Context::current().state().shader.completionStatusImplementation(_id, GL_COMPLETION_STATUS_KHR, &success);
+    return success == GL_TRUE;
+}
+
 void Shader::cleanLogImplementationNoOp(std::string&) {}
 
 #if defined(CORRADE_TARGET_WINDOWS) && !defined(MAGNUM_TARGET_GLES)
@@ -825,6 +820,10 @@ void Shader::cleanLogImplementationIntelWindows(std::string& message) {
 }
 #endif
 
+void Shader::completionStatusImplementationFallback(GLuint, GLenum, GLint* value) {
+    *value = GL_TRUE;
+}
+
 #ifndef DOXYGEN_GENERATING_OUTPUT
 Debug& operator<<(Debug& debug, const Shader::Type value) {
     debug << "GL::Shader::Type" << Debug::nospace;